From 57ba7ec14880658f7ac58f5d9f7a0148e0d35524 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Mon, 25 Nov 2024 18:04:25 +0000 Subject: [PATCH 1/7] Copy direct bazel starlark packages Not sure yet if we need more transitives, we'll see --- .../bazel/main/java/net/starlark/java/BUILD | 35 + .../main/java/net/starlark/java/annot/BUILD | 32 + .../java/net/starlark/java/annot/Param.java | 136 ++ .../net/starlark/java/annot/ParamType.java | 41 + .../java/net/starlark/java/annot/README.md | 4 + .../java/annot/StarlarkAnnotations.java | 200 +++ .../starlark/java/annot/StarlarkBuiltin.java | 80 ++ .../starlark/java/annot/StarlarkMethod.java | 206 +++ .../net/starlark/java/annot/processor/BUILD | 32 + .../processor/StarlarkMethodProcessor.java | 527 +++++++ .../main/java/net/starlark/java/eval/BUILD | 116 ++ .../starlark/java/eval/BuiltinFunction.java | 395 ++++++ .../net/starlark/java/eval/CallUtils.java | 234 +++ .../net/starlark/java/eval/CpuProfiler.java | 471 ++++++ .../java/net/starlark/java/eval/Debug.java | 176 +++ .../java/net/starlark/java/eval/Dict.java | 818 +++++++++++ .../java/net/starlark/java/eval/Eval.java | 899 ++++++++++++ .../net/starlark/java/eval/EvalException.java | 230 +++ .../net/starlark/java/eval/EvalUtils.java | 531 +++++++ .../starlark/java/eval/FlagGuardedValue.java | 87 ++ .../net/starlark/java/eval/FormatParser.java | 294 ++++ .../net/starlark/java/eval/GuardedValue.java | 49 + .../net/starlark/java/eval/HasBinary.java | 41 + .../eval/ImmutableSingletonStarlarkList.java | 78 + .../java/eval/ImmutableStarlarkList.java | 130 ++ .../main/java/net/starlark/java/eval/JNI.java | 30 + .../java/eval/LazyImmutableStarlarkList.java | 61 + .../starlark/java/eval/MethodDescriptor.java | 320 +++++ .../net/starlark/java/eval/MethodLibrary.java | 984 +++++++++++++ .../java/net/starlark/java/eval/Module.java | 321 +++++ .../net/starlark/java/eval/Mutability.java | 281 ++++ .../java/eval/MutableStarlarkList.java | 230 +++ .../java/net/starlark/java/eval/NoneType.java | 51 + .../starlark/java/eval/ParamDescriptor.java | 247 ++++ .../java/net/starlark/java/eval/Printer.java | 423 ++++++ .../net/starlark/java/eval/RangeList.java | 214 +++ .../eval/RegularImmutableStarlarkList.java | 42 + .../net/starlark/java/eval/RegularTuple.java | 159 +++ .../java/net/starlark/java/eval/Sequence.java | 157 ++ .../starlark/java/eval/SingletonTuple.java | 119 ++ .../java/net/starlark/java/eval/Starlark.java | 1179 +++++++++++++++ .../starlark/java/eval/StarlarkCallable.java | 99 ++ .../net/starlark/java/eval/StarlarkFloat.java | 290 ++++ .../starlark/java/eval/StarlarkFunction.java | 472 ++++++ .../starlark/java/eval/StarlarkIndexable.java | 54 + .../net/starlark/java/eval/StarlarkInt.java | 798 +++++++++++ .../starlark/java/eval/StarlarkIterable.java | 24 + .../net/starlark/java/eval/StarlarkList.java | 539 +++++++ .../java/eval/StarlarkMembershipTestable.java | 24 + .../starlark/java/eval/StarlarkSemantics.java | 262 ++++ .../net/starlark/java/eval/StarlarkSet.java | 777 ++++++++++ .../starlark/java/eval/StarlarkThread.java | 615 ++++++++ .../net/starlark/java/eval/StarlarkValue.java | 106 ++ .../net/starlark/java/eval/StringModule.java | 1067 ++++++++++++++ .../net/starlark/java/eval/Structure.java | 82 ++ .../starlark/java/eval/SymbolGenerator.java | 138 ++ .../java/net/starlark/java/eval/Tuple.java | 161 +++ .../starlark/java/eval/cpu_profiler_posix.cc | 203 +++ .../starlark/java/eval/cpu_profiler_unimpl.cc | 48 + .../java/net/starlark/java/lib/json/BUILD | 24 + .../java/net/starlark/java/lib/json/Json.java | 774 ++++++++++ .../java/net/starlark/java/spelling/BUILD | 27 + .../starlark/java/spelling/SpellChecker.java | 110 ++ .../net/starlark/java/syntax/Argument.java | 123 ++ .../java/syntax/AssignmentStatement.java | 85 ++ .../main/java/net/starlark/java/syntax/BUILD | 71 + .../java/syntax/BinaryOperatorExpression.java | 96 ++ .../starlark/java/syntax/CallExpression.java | 99 ++ .../net/starlark/java/syntax/Comment.java | 52 + .../starlark/java/syntax/Comprehension.java | 165 +++ .../java/syntax/ConditionalExpression.java | 57 + .../starlark/java/syntax/DefStatement.java | 91 ++ .../starlark/java/syntax/DictExpression.java | 93 ++ .../starlark/java/syntax/DotExpression.java | 56 + .../net/starlark/java/syntax/Expression.java | 76 + .../java/syntax/ExpressionStatement.java | 45 + .../starlark/java/syntax/FileLocations.java | 122 ++ .../net/starlark/java/syntax/FileOptions.java | 109 ++ .../starlark/java/syntax/FloatLiteral.java | 53 + .../starlark/java/syntax/FlowStatement.java | 56 + .../starlark/java/syntax/ForStatement.java | 81 ++ .../net/starlark/java/syntax/Identifier.java | 132 ++ .../net/starlark/java/syntax/IfStatement.java | 92 ++ .../starlark/java/syntax/IndexExpression.java | 67 + .../net/starlark/java/syntax/IntLiteral.java | 102 ++ .../java/syntax/LambdaExpression.java | 70 + .../java/net/starlark/java/syntax/Lexer.java | 962 +++++++++++++ .../starlark/java/syntax/ListExpression.java | 106 ++ .../starlark/java/syntax/LoadStatement.java | 85 ++ .../net/starlark/java/syntax/Location.java | 115 ++ .../java/net/starlark/java/syntax/Node.java | 128 ++ .../net/starlark/java/syntax/NodePrinter.java | 482 +++++++ .../net/starlark/java/syntax/NodeVisitor.java | 200 +++ .../net/starlark/java/syntax/Parameter.java | 147 ++ .../java/net/starlark/java/syntax/Parser.java | 1261 +++++++++++++++++ .../net/starlark/java/syntax/ParserInput.java | 109 ++ .../net/starlark/java/syntax/Program.java | 102 ++ .../net/starlark/java/syntax/Resolver.java | 1103 ++++++++++++++ .../starlark/java/syntax/ReturnStatement.java | 57 + .../starlark/java/syntax/SliceExpression.java | 82 ++ .../starlark/java/syntax/StarlarkFile.java | 168 +++ .../net/starlark/java/syntax/Statement.java | 50 + .../starlark/java/syntax/StringLiteral.java | 98 ++ .../net/starlark/java/syntax/SyntaxError.java | 99 ++ .../net/starlark/java/syntax/TokenKind.java | 112 ++ .../java/syntax/UnaryOperatorExpression.java | 63 + 106 files changed, 24876 insertions(+) create mode 100644 third_party/bazel/main/java/net/starlark/java/BUILD create mode 100644 third_party/bazel/main/java/net/starlark/java/annot/BUILD create mode 100644 third_party/bazel/main/java/net/starlark/java/annot/Param.java create mode 100644 third_party/bazel/main/java/net/starlark/java/annot/ParamType.java create mode 100644 third_party/bazel/main/java/net/starlark/java/annot/README.md create mode 100644 third_party/bazel/main/java/net/starlark/java/annot/StarlarkAnnotations.java create mode 100644 third_party/bazel/main/java/net/starlark/java/annot/StarlarkBuiltin.java create mode 100644 third_party/bazel/main/java/net/starlark/java/annot/StarlarkMethod.java create mode 100644 third_party/bazel/main/java/net/starlark/java/annot/processor/BUILD create mode 100644 third_party/bazel/main/java/net/starlark/java/annot/processor/StarlarkMethodProcessor.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/BUILD create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/BuiltinFunction.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/CallUtils.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/CpuProfiler.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/Debug.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/Dict.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/Eval.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/EvalException.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/EvalUtils.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/FlagGuardedValue.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/FormatParser.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/GuardedValue.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/HasBinary.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/ImmutableSingletonStarlarkList.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/ImmutableStarlarkList.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/JNI.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/LazyImmutableStarlarkList.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/MethodDescriptor.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/MethodLibrary.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/Module.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/Mutability.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/MutableStarlarkList.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/NoneType.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/ParamDescriptor.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/Printer.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/RangeList.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/RegularImmutableStarlarkList.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/RegularTuple.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/Sequence.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/SingletonTuple.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/Starlark.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/StarlarkCallable.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/StarlarkFloat.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/StarlarkFunction.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/StarlarkIndexable.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/StarlarkInt.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/StarlarkIterable.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/StarlarkList.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/StarlarkMembershipTestable.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/StarlarkSemantics.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/StarlarkSet.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/StarlarkThread.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/StarlarkValue.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/StringModule.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/Structure.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/SymbolGenerator.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/Tuple.java create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/cpu_profiler_posix.cc create mode 100644 third_party/bazel/main/java/net/starlark/java/eval/cpu_profiler_unimpl.cc create mode 100644 third_party/bazel/main/java/net/starlark/java/lib/json/BUILD create mode 100644 third_party/bazel/main/java/net/starlark/java/lib/json/Json.java create mode 100644 third_party/bazel/main/java/net/starlark/java/spelling/BUILD create mode 100644 third_party/bazel/main/java/net/starlark/java/spelling/SpellChecker.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/Argument.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/AssignmentStatement.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/BUILD create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/BinaryOperatorExpression.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/CallExpression.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/Comment.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/Comprehension.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/ConditionalExpression.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/DefStatement.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/DictExpression.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/DotExpression.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/Expression.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/ExpressionStatement.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/FileLocations.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/FileOptions.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/FloatLiteral.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/FlowStatement.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/ForStatement.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/Identifier.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/IfStatement.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/IndexExpression.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/IntLiteral.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/LambdaExpression.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/Lexer.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/ListExpression.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/LoadStatement.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/Location.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/Node.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/NodePrinter.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/NodeVisitor.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/Parameter.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/Parser.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/ParserInput.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/Program.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/Resolver.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/ReturnStatement.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/SliceExpression.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/StarlarkFile.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/Statement.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/StringLiteral.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/SyntaxError.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/TokenKind.java create mode 100644 third_party/bazel/main/java/net/starlark/java/syntax/UnaryOperatorExpression.java diff --git a/third_party/bazel/main/java/net/starlark/java/BUILD b/third_party/bazel/main/java/net/starlark/java/BUILD new file mode 100644 index 000000000..06d180dc3 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/BUILD @@ -0,0 +1,35 @@ +# Bazel's Starlark interpreter + +licenses(["notice"]) + +filegroup( + name = "srcs", + srcs = ["BUILD"], + visibility = [":bazel"], +) + +# The java.starlark.net project, including its tests. +package_group( + name = "starlark", + packages = [ + "//src/main/java/net/starlark/java/...", + "//src/test/java/net/starlark/java/...", + ], +) + +# Bazel and its tests. +package_group( + name = "bazel", + packages = ["//src/..."], +) + +# The Java Starlark interpreter is not supported for general use as a public +# API. However, Copybara and Stardoc are approved clients. +# +# We do not use visibility to prevent other dependencies, because cross-repo +# visibility restrictions do not make sense. Just be aware this is not +# guaranteed as a stable API. +package_group( + name = "clients", + packages = ["public"], +) diff --git a/third_party/bazel/main/java/net/starlark/java/annot/BUILD b/third_party/bazel/main/java/net/starlark/java/annot/BUILD new file mode 100644 index 000000000..d9bed9bf5 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/annot/BUILD @@ -0,0 +1,32 @@ +load("@rules_java//java:defs.bzl", "java_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src:__subpackages__"], +) + +# Annotations to make Java classes and methods accessible from Starlark. +# Depending on this library adds annotations processing to the build. +java_library( + name = "annot", + exported_plugins = ["//src/main/java/net/starlark/java/annot/processor"], + visibility = ["//src/main/java/net/starlark/java:clients"], + exports = [":annot_sans_processor"], +) + +# Annotations without processor, to avoid a dependency cycle in the processor itself. +java_library( + name = "annot_sans_processor", + srcs = glob(["*.java"]), + visibility = ["//src/main/java/net/starlark/java/annot/processor:__pkg__"], + deps = [ + "//third_party:guava", + "//third_party:jsr305", + ], +) diff --git a/third_party/bazel/main/java/net/starlark/java/annot/Param.java b/third_party/bazel/main/java/net/starlark/java/annot/Param.java new file mode 100644 index 000000000..5bae876cd --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/annot/Param.java @@ -0,0 +1,136 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.annot; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** An annotation for parameters of Starlark built-in functions. */ +@Retention(RetentionPolicy.RUNTIME) +public @interface Param { + + /** + * Name of the parameter, as viewed from Starlark. Used for matching keyword arguments and for + * generating documentation. + */ + String name(); + + /** + * Documentation of the parameter. + */ + String doc() default ""; + + /** + * Determines whether the parameter appears in generated documentation. Set this to false to + * suppress parameters whose use is intentionally restricted. + * + *

An undocumented parameter must be {@link #named} and may not be followed by positional + * parameters or {@code **kwargs}. + */ + boolean documented() default true; + + /** + * Default value for the parameter, written as a Starlark expression (e.g. "False", "True", "[]", + * "None"). + * + *

If this is empty (the default), the parameter is treated as mandatory. (Thus an exception + * will be thrown if left unspecified by the caller). + * + *

If the function implementation needs to distinguish the case where the caller does not + * supply a value for this parameter, you can set the default to the magic string "unbound", which + * maps to the sentinal object {@link net.starlark.java.eval.Starlark#UNBOUND} (which can't appear + * in normal Starlark code). + */ + String defaultValue() default ""; + + /** + * List of allowed types for the parameter. + * + *

The array may be omitted, in which case the parameter accepts any value whose class is + * assignable to the class of the parameter variable. + * + *

If a function should accept None, NoneType should be in this list. + */ + ParamType[] allowedTypes() default {}; + + /** + * If true, the parameter may be specified as a named parameter. For example for an integer named + * parameter {@code foo} of a method {@code bar}, then the method call will look like {@code + * bar(foo=1)}. + * + *

If false, then {@link #positional} must be true (otherwise there is no way to reference the + * parameter via an argument). + * + *

If this parameter represents the 'extra positionals' (args) or 'extra keywords' (kwargs) + * element of a method, this field has no effect. + */ + boolean named() default false; + + /** + * If true, the parameter may be specified as a positional parameter. For example for an integer + * positional parameter {@code foo} of a method {@code bar}, then the method call will look like + * {@code bar(1)}. If {@link #named()} is {@code false}, then this will be the only way to call + * {@code bar}. + * + *

If false, then {@link #named} must be true (otherwise there is no way to reference the + * parameter via an argument) + * + *

Positional arguments should come first. + * + *

If this parameter represents the 'extra positionals' (args) or 'extra keywords' (kwargs) + * element of a method, this field has no effect. + */ + boolean positional() default true; + + /** + * If non-empty, the annotated parameter will only be present if the given semantic flag is true. + * (If the parameter is disabled, it may not be specified by a user, and the Java method will + * always be invoked with the parameter set to its default value.) + * + *

Note that at most one of {@link #enableOnlyWithFlag} and {@link #disableWithFlag} can be + * non-empty. + */ + String enableOnlyWithFlag() default ""; + + /** + * If non-empty, the annotated parameter will only be present if the given semantic flag is false. + * (If the parameter is disabled, it may not be specified by a user, and the Java method will + * always be invoked with the parameter set to its default value.) + * + *

Note that at most one of {@link #enableOnlyWithFlag} and {@link #disableWithFlag} can be + * non-empty. + */ + String disableWithFlag() default ""; + + /** + * Value for the parameter when the parameter is "disabled" based on semantic flags. (When the + * parameter is disabled, it may not be set from Starlark, but an argument of the given value is + * passed to the annotated Java method when invoked.) (See {@link #enableOnlyWithFlag()} and + * {@link #disableWithFlag()} for toggling a parameter with semantic flags. + * + *

The parameter value is written as a Starlark expression (for example: "False", "True", "[]", + * "None"). + * + *

This should be set (non-empty) if and only if the parameter may be disabled with a semantic + * flag. + * + *

Note that this is very similar to {@link #defaultValue}; it may be considered "the default + * value if no parameter is specified". It is important that this is distinct, however, in cases + * where it is desired to have a normally-mandatory parameter toggled by flag. Such a parameter + * should have no {@link #defaultValue} set, but should have a sensible {@link + * #valueWhenDisabled()} set. ("unbound" may be used in cases where no value would be valid. See + * {@link #defaultValue}.) + */ + String valueWhenDisabled() default ""; +} diff --git a/third_party/bazel/main/java/net/starlark/java/annot/ParamType.java b/third_party/bazel/main/java/net/starlark/java/annot/ParamType.java new file mode 100644 index 000000000..f4fc5dff0 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/annot/ParamType.java @@ -0,0 +1,41 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.annot; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** An annotation for parameter types for Starlark built-in functions. */ +@Retention(RetentionPolicy.RUNTIME) +public @interface ParamType { + /** + * The Java class of the type, e.g. {@link String}.class or {@link + * net.starlark.java.eval.Sequence}.class. + */ + Class type(); + + /** + * When {@link #type()} is a generic type (e.g., {@link net.starlark.java.eval.Sequence}), specify + * the type parameter (e.g. {@link String}.class} along with {@link + * net.starlark.java.eval.Sequence} for {@link #type()} to specify a list of strings). + * + *

This is only used for documentation generation. The actual generic type is not checked at + * runtime, so the Java method signature should use a generic type of Object and cast + * appropriately. + */ + // TODO(#13365): make this a data structure so we can represent a {@link + // net.starlark.java.eval.Sequence} of types {@code A} or {@code B} intermixed, a {@link + // net.starlark.java.eval.Dict} mapping from {@code A} to {@code B}, etc. + Class generic1() default Object.class; +} diff --git a/third_party/bazel/main/java/net/starlark/java/annot/README.md b/third_party/bazel/main/java/net/starlark/java/annot/README.md new file mode 100644 index 000000000..a03140b83 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/annot/README.md @@ -0,0 +1,4 @@ +# Starlark interface framework + +The classes in this package define annotations and interfaces used to enable +Starlark access to data types and methods implemented in Java. diff --git a/third_party/bazel/main/java/net/starlark/java/annot/StarlarkAnnotations.java b/third_party/bazel/main/java/net/starlark/java/annot/StarlarkAnnotations.java new file mode 100644 index 000000000..6fbf56ba4 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/annot/StarlarkAnnotations.java @@ -0,0 +1,200 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.annot; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import javax.annotation.Nullable; + +/** Utility functions for Starlark annotations. */ +public final class StarlarkAnnotations { + + /** + * Returns the more specific class of two classes. Class x is more specific than class y if x is + * assignable to y. For example, of String.class and Object.class, String.class is more specific. + * + *

If either class is null, returns the other class. + * + *

If the classes are identical, returns the class. + * + * @throws IllegalArgumentException if neither class is assignable to the other + */ + private static Class moreSpecific(Class x, Class y) { + if (x == null) { + return y; + } else if (y == null) { + return x; + } else if (x.isAssignableFrom(y)) { + return y; + } else if (y.isAssignableFrom(x)) { + return x; + } else { + // If this exception occurs, it indicates the following error scenario: + // + // Suppose class A is a subclass of both B and C, where B and C are annotated with + // @StarlarkBuiltin annotations (and are thus considered "Starlark types"). If B is not a + // subclass of C (nor vice versa), then it's impossible to resolve whether A is of type + // B or if A is of type C. It's both! The way to resolve this is usually to have A be its own + // type (annotated with @StarlarkBuiltin), and thus have the explicit type of A be + // semantically "B and C". + throw new IllegalArgumentException( + String.format("Expected one of %s and %s to be a subclass of the other", x, y)); + } + } + + /** + * Searches a class or interface's class hierarchy for the given class annotation. + * + *

If the given class annotation appears multiple times within the class hierachy, this chooses + * the annotation on the most-specified class in the hierarchy. + * + * @return the best-fit class that declares the annotation, or null if no class in the hierarchy + * declares it + * @throws IllegalArgumentException if the most-specified class in the hierarchy having the + * annotation is not unique + */ + @Nullable + private static Class findAnnotatedAncestor( + Class classObj, Class annotation) { + if (classObj.isAnnotationPresent(annotation)) { + return classObj; + } + Class bestCandidate = null; + Class superclass = classObj.getSuperclass(); + if (superclass != null) { + Class result = findAnnotatedAncestor(superclass, annotation); + bestCandidate = moreSpecific(result, bestCandidate); + } + for (Class interfaceObj : classObj.getInterfaces()) { + Class result = findAnnotatedAncestor(interfaceObj, annotation); + bestCandidate = moreSpecific(result, bestCandidate); + } + return bestCandidate; + } + + /** + * Returns the {@link StarlarkBuiltin} annotation for the given class, if it exists, and + * null otherwise. The first annotation found will be returned, starting with {@code classObj} + * and following its base classes and interfaces recursively. + */ + @Nullable + public static StarlarkBuiltin getStarlarkBuiltin(Class classObj) { + Class cls = findAnnotatedAncestor(classObj, StarlarkBuiltin.class); + return cls == null ? null : cls.getAnnotation(StarlarkBuiltin.class); + } + + /** + * Searches {@code classObj}'s class hierarchy and returns the first superclass or interface that + * is annotated with {@link StarlarkBuiltin} (including possibly {@code classObj} itself), or null + * if none is found. + */ + @Nullable + public static Class getParentWithStarlarkBuiltin(Class classObj) { + return findAnnotatedAncestor(classObj, StarlarkBuiltin.class); + } + + /** + * Returns the {@link StarlarkMethod} annotation for the given method, if it exists, and null + * otherwise. + * + *

Note that the annotation may be defined on a supermethod, rather than directly on the given + * method. + * + *

{@code classObj} is the class on which the given method is defined. + */ + @Nullable + public static StarlarkMethod getStarlarkMethod(Class classObj, Method method) { + StarlarkMethod callable = getAnnotationOnClassMatchingSignature(classObj, method); + if (callable != null) { + return callable; + } + if (classObj.getSuperclass() != null) { + StarlarkMethod annotation = getStarlarkMethod(classObj.getSuperclass(), method); + if (annotation != null) { + return annotation; + } + } + for (Class interfaceObj : classObj.getInterfaces()) { + StarlarkMethod annotation = getStarlarkMethod(interfaceObj, method); + if (annotation != null) { + return annotation; + } + } + return null; + } + + /** + * Convenience version of {@code getAnnotationsFromParentClass(Class, Method)} that uses the + * declaring class of the method. + */ + @Nullable + public static StarlarkMethod getStarlarkMethod(Method method) { + return getStarlarkMethod(method.getDeclaringClass(), method); + } + + /** + * Returns the {@code StarlarkMethod} annotation corresponding to the given method of the given + * class, or null if there is no such annotation. + * + *

This method checks assignability instead of exact matches for purposes of generics. If Clazz + * has parameters BarT (extends BarInterface) and BazT (extends BazInterface), then foo(BarT, + * BazT) should match if the given method signature is foo(BarImpl, BazImpl). The signatures are + * in inexact match, but an "assignable" match. + */ + @Nullable + private static StarlarkMethod getAnnotationOnClassMatchingSignature( + Class classObj, Method signatureToMatch) { + // TODO(b/79877079): This method validates several invariants of @StarlarkMethod. These + // invariants should be verified in annotation processor or in test, and left out of this + // method. + Method[] methods = classObj.getDeclaredMethods(); + Class[] paramsToMatch = signatureToMatch.getParameterTypes(); + + StarlarkMethod callable = null; + + for (Method method : methods) { + if (signatureToMatch.getName().equals(method.getName()) + && method.isAnnotationPresent(StarlarkMethod.class)) { + Class[] paramTypes = method.getParameterTypes(); + + if (paramTypes.length == paramsToMatch.length) { + for (int i = 0; i < paramTypes.length; i++) { + // This verifies assignability of the method signature to ensure this is not a + // coincidental overload. We verify assignability instead of matching exact parameter + // classes in order to match generic methods. + if (!paramTypes[i].isAssignableFrom(paramsToMatch[i])) { + throw new IllegalStateException( + String.format( + "Class %s has an incompatible overload of annotated method %s declared by %s", + classObj, signatureToMatch.getName(), signatureToMatch.getDeclaringClass())); + } + } + } + if (callable == null) { + callable = method.getAnnotation(StarlarkMethod.class); + } else { + throw new IllegalStateException( + String.format( + "Class %s has multiple overloaded methods named '%s' annotated " + + "with @StarlarkMethod", + classObj, signatureToMatch.getName())); + } + } + } + return callable; + } + + private StarlarkAnnotations() {} +} diff --git a/third_party/bazel/main/java/net/starlark/java/annot/StarlarkBuiltin.java b/third_party/bazel/main/java/net/starlark/java/annot/StarlarkBuiltin.java new file mode 100644 index 000000000..cbdc76a34 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/annot/StarlarkBuiltin.java @@ -0,0 +1,80 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.annot; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * This annotation is used on classes and interfaces that represent Starlark data types. + * + *

Conceptually, every {@link StarlarkBuiltin} annotation corresponds to a user-distinguishable + * Starlark type. The annotation holds metadata associated with that type, in particular its name + * and documentation. The annotation also implicitly demarcates the Starlark API of the type. It + * does not matter whether the annotation is used on a class or an interface. + * + *

Annotations are "inherited" and "overridden", in the sense that a child class or interface + * takes on the Starlark type of its ancestor by default, unless it has a direct annotation of its + * own. If there are multiple ancestors that have an annotation, then to avoid ambiguity we require + * that one of them is a subtype of the rest; that is the one whose annotation gets inherited. This + * ensures that every class implements at most one Starlark type, and not an ad hoc hybrid of + * multiple types. (In mathematical terms, the most-derived annotation for class or interface C is + * the minimum element in the partial order of all annotations defined on C and its ancestors, where + * the order relationship is X < Y if X annotates a subtype of what Y annotates.) The lookup logic + * for retrieving a class's {@link StarlarkBuiltin} is implemented by {@link + * StarlarkAnnotations#getStarlarkBuiltin}. + * + *

Inheriting an annotation is useful when the class is an implementation detail, such as a + * concrete implementation of an abstract interface. Overriding an annotation is useful when the + * class should have its own distinct user-visible API or documentation. For example, {@link + * Sequence} is an abstract type implemented by both {@link StarlarkList} and {@link + * Sequence.Tuple}, all three of which are annotated. Annotating the list and tuple types allows + * them to define different methods, while annotating {@link Sequence} allows them to be identified + * as a single type for the purpose of type checking, documentation, and error messages. + * + *

All {@link StarlarkBuiltin}-annotated types must implement {@link StarlarkValue}. Nearly all + * non-abstract implementations of {@link StarlarkValue} have or inherit a {@link StarlarkBuiltin} + * annotation. (It is possible, though quite unusual, to declare an implementation of {@code + * StarlarkValue} without using the annotation mechanism defined in this package. {@code + * StarlarkFunction} is one example.) + */ +// TODO(adonovan): rename to StarlarkType now that that name is available again. +@Target({ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +public @interface StarlarkBuiltin { + + /** + * The name of this data type, as returned by the Starlark expression {@code type(x)}. + * + *

Applications should ensure that data type names are unique. This is especially important for + * a type that implements Comparable, as its {@code compareTo} method may be passed any value of + * the same Starlark type, not necessarily one of the same Java class. + */ + String name(); + + /** Module documentation in HTML. May be empty only if {@code !documented()}. */ + String doc() default ""; + + /** Whether the module should appear in the documentation. */ + boolean documented() default true; + + /** + * The category of the documentation to which this data type belongs. Applications may use this + * field as they wish for their documentation tools. The core data types of the Starlark + * interpreter all have category "core". + */ + String category() default ""; +} diff --git a/third_party/bazel/main/java/net/starlark/java/annot/StarlarkMethod.java b/third_party/bazel/main/java/net/starlark/java/annot/StarlarkMethod.java new file mode 100644 index 000000000..bedc13575 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/annot/StarlarkMethod.java @@ -0,0 +1,206 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.annot; + +import com.google.errorprone.annotations.Keep; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotates a Java method that can be called from Starlark. + * + *

A method annotated with {@code @StarlarkMethod} may not have overloads or hide any static or + * default methods. Overriding is allowed, but the {@code @StarlarkMethod} annotation itself must + * not be repeated on the override. This ensures that given a method, we can always determine its + * corresponding {@code @StarlarkMethod} annotation, if it has one, by scanning all methods of the + * same name in its class hierarchy, without worrying about complications like overloading or + * generics. The lookup functionality is implemented by {@link + * StarlarkAnnotations#getStarlarkMethod}. + * + *

Methods having this annotation must satisfy the following requirements, which are enforced at + * compile time by {@link StarlarkMethodProcessor}: + * + *

+ * + *

When an annotated method is called from Starlark, it is a dynamic error if it returns null, + * unless the method is marked as {@link #allowReturnNones}, in which case {@link Starlark#fromJava} + * converts the Java null value to {@link Starlark#NONE}. This feature prevents a method whose + * declared (and documented) result type is T from unexpectedly returning a value of type NoneType. + * + *

The annotated method may throw any checked or unchecked exceptions. When it is invoked, + * unchecked exceptions, {@code EvalException}s, and {@code InterruptedException}s are passed + * through; all other (checked) exceptions are wrapped in an {@code EvalException} and thrown. + */ +// TODO(adonovan): rename to StarlarkAttribute and factor Starlark{Method,Field} as subinterfaces. +@Keep +@Target({ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface StarlarkMethod { + + /** Name of the method, as exposed to Starlark. */ + String name(); + + /** + * The documentation text in Starlark. It can contain HTML tags for special formatting. + * + *

It is allowed to be empty only if {@link #documented()} is false. + */ + String doc() default ""; + + /** + * If true, the function will appear in the Starlark documentation. Set this to false if the + * function is experimental or an overloading and doesn't need to be documented. + */ + boolean documented() default true; + + /** + * If true, this method will be considered as a field of the enclosing Java object. E.g., if set + * to true on a method {@code foo}, then the callsites of this method will look like {@code + * bar.foo} instead of {@code bar.foo()}. The annotated method must be parameterless and {@link + * #parameters()} should be empty. + */ + boolean structField() default false; + + /** + * List of parameters this function accepts. + */ + Param[] parameters() default {}; + + /** + * Defines a catch-all list for additional unspecified positional parameters. + * + *

If this is left as default, it is an error for the caller to pass more positional arguments + * than are explicitly allowed by the method signature. If this is defined, all additional + * positional arguments are passed as elements of a {@link Tuple} to the method. + * + *

See Python's *args (http://thepythonguru.com/python-args-and-kwargs/). + * + *

If defined, the annotated method must declare a corresponding parameter to which a {@code + * Tuple} may be assigned. See the interface-level javadoc for details. + */ + // TODO(adonovan): consider using a simpler type than Param here. All that's needed at run-time + // is a boolean. The doc tools want a name and doc string, but the rest is irrelevant and + // distracting. + // Ditto extraKeywords. + Param extraPositionals() default @Param(name = ""); + + /** + * Defines a catch-all dictionary for additional unspecified named parameters. + * + *

If this is left as default, it is an error for the caller to pass any named arguments not + * explicitly declared by the method signature. If this is defined, all additional named arguments + * are passed as elements of a {@code Dict} to the method. + * + *

See Python's **kwargs (http://thepythonguru.com/python-args-and-kwargs/). + * + *

If defined, the annotated method must declare a corresponding parameter to which a {@code + * Dict} may be assigned. See the interface-level javadoc for details. + */ + Param extraKeywords() default @Param(name = ""); + + /** + * If true, indicates that the class containing the annotated method has the ability to be called + * from Starlark (as if it were a function) and that the annotated method should be invoked when + * this occurs. + * + *

A class may only have one method with selfCall set to true. + * + *

A method with selfCall=true must not be a structField, and must have name specified (used + * for descriptive errors if, for example, there are missing arguments). + */ + boolean selfCall() default false; + + /** + * Permits the Java method to return null, which {@link Starlark#fromJava} then converts to {@link + * Starlark#NONE}. If false, a null result causes the Starlark call to fail. + */ + boolean allowReturnNones() default false; + + /** + * If true, the StarlarkThread will be passed as an argument of the annotated function. (Thus, the + * annotated method signature must contain StarlarkThread as a parameter. See the interface-level + * javadoc for details.) + * + *

This is incompatible with structField=true. If structField is true, this must be false. + */ + boolean useStarlarkThread() default false; + + /** + * If true, the Starlark semantics will be passed to the annotated Java method. (Thus, the + * annotated method signature must contain StarlarkSemantics as a parameter. See the + * interface-level javadoc for details.) + * + *

This option is allowed only for fields ({@code structField=true}). For methods, the {@code + * StarlarkThread} parameter provides access to the semantics, and more. + */ + boolean useStarlarkSemantics() default false; + + /** + * If non-empty, the annotated method will only be callable if the given semantic flag is true. + * Note that at most one of {@link #enableOnlyWithFlag} and {@link #disableWithFlag} can be + * non-empty. + */ + String enableOnlyWithFlag() default ""; + + /** + * If non-empty, the annotated method will only be callable if the given semantic flag is false. + * Note that at most one of {@link #enableOnlyWithFlag} and {@link #disableWithFlag} can be + * non-empty. + */ + String disableWithFlag() default ""; +} diff --git a/third_party/bazel/main/java/net/starlark/java/annot/processor/BUILD b/third_party/bazel/main/java/net/starlark/java/annot/processor/BUILD new file mode 100644 index 000000000..d9fd1925a --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/annot/processor/BUILD @@ -0,0 +1,32 @@ +load("@rules_java//java:defs.bzl", "java_library", "java_plugin") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src:__subpackages__"], +) + +# A processor for Starlark annotations. +java_plugin( + name = "processor", + processor_class = "net.starlark.java.annot.processor.StarlarkMethodProcessor", + visibility = ["//src/main/java/net/starlark/java:starlark"], + deps = [":processor_lib"], +) + +java_library( + name = "processor_lib", + srcs = ["StarlarkMethodProcessor.java"], + visibility = ["//src/main/java/net/starlark/java:starlark"], + deps = [ + "//src/main/java/net/starlark/java/annot:annot_sans_processor", + "//third_party:error_prone_annotations", + "//third_party:guava", + "//third_party:jsr305", + ], +) diff --git a/third_party/bazel/main/java/net/starlark/java/annot/processor/StarlarkMethodProcessor.java b/third_party/bazel/main/java/net/starlark/java/annot/processor/StarlarkMethodProcessor.java new file mode 100644 index 000000000..c2406f056 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/annot/processor/StarlarkMethodProcessor.java @@ -0,0 +1,527 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.annot.processor; + +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.SetMultimap; +import com.google.errorprone.annotations.FormatMethod; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import javax.annotation.Nullable; +import javax.annotation.processing.AbstractProcessor; +import javax.annotation.processing.Messager; +import javax.annotation.processing.ProcessingEnvironment; +import javax.annotation.processing.RoundEnvironment; +import javax.annotation.processing.SupportedAnnotationTypes; +import javax.lang.model.SourceVersion; +import javax.lang.model.element.Element; +import javax.lang.model.element.ExecutableElement; +import javax.lang.model.element.Modifier; +import javax.lang.model.element.TypeElement; +import javax.lang.model.element.VariableElement; +import javax.lang.model.type.DeclaredType; +import javax.lang.model.type.MirroredTypeException; +import javax.lang.model.type.TypeKind; +import javax.lang.model.type.TypeMirror; +import javax.lang.model.type.WildcardType; +import javax.lang.model.util.Elements; +import javax.lang.model.util.Types; +import javax.tools.Diagnostic; +import net.starlark.java.annot.Param; +import net.starlark.java.annot.ParamType; +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; + +/** + * Annotation processor for {@link StarlarkMethod}. See that class for requirements. + * + *

These properties can be relied upon at runtime without additional checks. + */ +@SupportedAnnotationTypes({ + "net.starlark.java.annot.StarlarkMethod", + "net.starlark.java.annot.StarlarkBuiltin" +}) +public class StarlarkMethodProcessor extends AbstractProcessor { + + private Types types; + private Elements elements; + private Messager messager; + + // A set containing a TypeElement for each class with a StarlarkMethod.selfCall annotation. + private Set classesWithSelfcall; + // A multimap where keys are class element, and values are the callable method names identified in + // that class (where "method name" is StarlarkMethod.name). + private SetMultimap processedClassMethods; + + @Override + public SourceVersion getSupportedSourceVersion() { + return SourceVersion.latestSupported(); + } + + @Override + public synchronized void init(ProcessingEnvironment env) { + super.init(env); + this.types = env.getTypeUtils(); + this.elements = env.getElementUtils(); + this.messager = env.getMessager(); + this.classesWithSelfcall = new HashSet<>(); + this.processedClassMethods = LinkedHashMultimap.create(); + } + + private TypeMirror getType(String canonicalName) { + return elements.getTypeElement(canonicalName).asType(); + } + + @Override + public boolean process(Set annotations, RoundEnvironment roundEnv) { + TypeMirror stringType = getType("java.lang.String"); + TypeMirror integerType = getType("java.lang.Integer"); + TypeMirror booleanType = getType("java.lang.Boolean"); + TypeMirror listType = getType("java.util.List"); + TypeMirror mapType = getType("java.util.Map"); + TypeMirror starlarkValueType = getType("net.starlark.java.eval.StarlarkValue"); + + // Ensure StarlarkBuiltin-annotated classes implement StarlarkValue. + for (Element cls : roundEnv.getElementsAnnotatedWith(StarlarkBuiltin.class)) { + if (!types.isAssignable(cls.asType(), starlarkValueType)) { + errorf( + cls, + "class %s has StarlarkBuiltin annotation but does not implement StarlarkValue", + cls.getSimpleName()); + } + } + + for (Element element : roundEnv.getElementsAnnotatedWith(StarlarkMethod.class)) { + // Only methods are annotated with StarlarkMethod. + // This is ensured by the @Target(ElementType.METHOD) annotation. + ExecutableElement method = (ExecutableElement) element; + if (!method.getModifiers().contains(Modifier.PUBLIC)) { + errorf(method, "StarlarkMethod-annotated methods must be public."); + } + if (method.getModifiers().contains(Modifier.STATIC)) { + errorf(method, "StarlarkMethod-annotated methods cannot be static."); + } + + // Check the annotation itself. + StarlarkMethod annot = method.getAnnotation(StarlarkMethod.class); + if (annot.name().isEmpty()) { + errorf(method, "StarlarkMethod.name must be non-empty."); + } + Element cls = method.getEnclosingElement(); + if (!processedClassMethods.put(cls, annot.name())) { + errorf(method, "Containing class defines more than one method named '%s'.", annot.name()); + } + if (annot.documented() && annot.doc().isEmpty()) { + errorf(method, "The 'doc' string must be non-empty if 'documented' is true."); + } + if (annot.structField()) { + checkStructFieldAnnotation(method, annot); + } else if (annot.useStarlarkSemantics()) { + errorf( + method, + "a StarlarkMethod-annotated method with structField=false may not also specify" + + " useStarlarkSemantics. (Instead, set useStarlarkThread and call" + + " getSemantics().)"); + } + if (annot.selfCall() && !classesWithSelfcall.add(cls)) { + errorf(method, "Containing class has more than one selfCall method defined."); + } + + boolean hasFlag = false; + if (!annot.enableOnlyWithFlag().isEmpty()) { + if (!hasPlusMinusPrefix(annot.enableOnlyWithFlag())) { + errorf(method, "enableOnlyWithFlag name must have a + or - prefix"); + } + hasFlag = true; + } + if (!annot.disableWithFlag().isEmpty()) { + if (!hasPlusMinusPrefix(annot.disableWithFlag())) { + errorf(method, "disableWithFlag name must have a + or - prefix"); + } + if (hasFlag) { + errorf( + method, + "Only one of StarlarkMethod.enableOnlyWithFlag and StarlarkMethod.disableWithFlag" + + " may be specified."); + } + hasFlag = true; + } + + if (annot.allowReturnNones() != (method.getAnnotation(Nullable.class) != null)) { + errorf(method, "Method must be annotated with @Nullable iff allowReturnNones is set."); + } + + checkParameters(method, annot); + + // Verify that result type, if final, might satisfy Starlark.fromJava. + // (If the type is non-final we can't prove that all subclasses are invalid.) + TypeMirror ret = method.getReturnType(); + if (ret.getKind() == TypeKind.DECLARED) { + DeclaredType obj = (DeclaredType) ret; + if (obj.asElement().getModifiers().contains(Modifier.FINAL) + && !types.isSameType(ret, stringType) + && !types.isSameType(ret, integerType) + && !types.isSameType(ret, booleanType) + && !types.isAssignable(obj, starlarkValueType) + && !types.isAssignable(obj, listType) + && !types.isAssignable(obj, mapType)) { + errorf( + method, + "StarlarkMethod-annotated method %s returns %s, which has no legal Starlark values" + + " (see Starlark.fromJava)", + method.getSimpleName(), + ret); + } + } + } + + // Returning false allows downstream processors to work on the same annotations + return false; + } + + // TODO(adonovan): obviate these checks by separating field/method interfaces. + private void checkStructFieldAnnotation(ExecutableElement method, StarlarkMethod annot) { + // useStructField is incompatible with special thread-related parameters, + // because unlike a method, which is actively called within a thread, + // a field is a passive part of a data structure that may be accessed + // from Java threads that don't have anything to do with Starlark threads. + // However, the StarlarkSemantics is available even to fields, + // because it is a required parameter for all attribute-selection + // operations x.f. + // + // Not having a thread forces implementations to assume Mutability=null, + // which is not quite right. Perhaps one day we can abolish Mutability + // in favor of a tracing approach as in go.starlark.net. + if (annot.useStarlarkThread()) { + errorf( + method, + "a StarlarkMethod-annotated method with structField=true may not also specify" + + " useStarlarkThread"); + } + if (!annot.extraPositionals().name().isEmpty()) { + errorf( + method, + "a StarlarkMethod-annotated method with structField=true may not also specify" + + " extraPositionals"); + } + if (!annot.extraKeywords().name().isEmpty()) { + errorf( + method, + "a StarlarkMethod-annotated method with structField=true may not also specify" + + " extraKeywords"); + } + if (annot.selfCall()) { + errorf( + method, + "a StarlarkMethod-annotated method with structField=true may not also specify" + + " selfCall=true"); + } + int nparams = annot.parameters().length; + if (nparams > 0) { + errorf( + method, + "method %s is annotated structField=true but also has %d Param annotations", + method.getSimpleName(), + nparams); + } + } + + private void checkParameters(ExecutableElement method, StarlarkMethod annot) { + List params = method.getParameters(); + + boolean allowPositionalNext = true; + boolean allowPositionalOnlyNext = true; + boolean allowNonDefaultPositionalNext = true; + boolean hasUndocumentedMethods = false; + + // Check @Param annotations match parameters. + Param[] paramAnnots = annot.parameters(); + for (int i = 0; i < paramAnnots.length; i++) { + Param paramAnnot = paramAnnots[i]; + if (i >= params.size()) { + errorf( + method, + "method %s has %d Param annotations but only %d parameters", + method.getSimpleName(), + paramAnnots.length, + params.size()); + return; + } + VariableElement param = params.get(i); + + checkParameter(param, paramAnnot); + + // Check parameter ordering. + if (paramAnnot.positional()) { + if (!allowPositionalNext) { + errorf( + param, + "Positional parameter '%s' is specified after one or more non-positional parameters", + paramAnnot.name()); + } + if (!paramAnnot.named() && !allowPositionalOnlyNext) { + errorf( + param, + "Positional-only parameter '%s' is specified after one or more named or undocumented" + + " parameters", + paramAnnot.name()); + } + if (paramAnnot.defaultValue().isEmpty()) { // There is no default value. + if (!allowNonDefaultPositionalNext) { + errorf( + param, + "Positional parameter '%s' has no default value but is specified after one " + + "or more positional parameters with default values", + paramAnnot.name()); + } + } else { // There is a default value. + // No positional parameters without a default value can come after this parameter. + allowNonDefaultPositionalNext = false; + } + } else { // Not positional. + // No positional parameters can come after this parameter. + allowPositionalNext = false; + + if (!paramAnnot.named()) { + errorf(param, "Parameter '%s' must be either positional or named", paramAnnot.name()); + } + } + if (!paramAnnot.documented()) { + hasUndocumentedMethods = true; + } + if (paramAnnot.named() || !paramAnnot.documented()) { + // No positional-only parameters can come after this parameter. + allowPositionalOnlyNext = false; + } + } + + if (hasUndocumentedMethods && !annot.extraKeywords().name().isEmpty()) { + errorf( + method, + "Method '%s' has undocumented parameters but also allows extra keyword parameters", + annot.name()); + } + checkSpecialParams(method, annot); + } + + // Checks consistency of a single parameter with its Param annotation. + private void checkParameter(Element param, Param paramAnnot) { + TypeMirror paramType = param.asType(); // type of the Java method parameter + + // Give helpful hint for parameter of type Integer. + TypeMirror integerType = getType("java.lang.Integer"); + if (types.isSameType(paramType, integerType)) { + errorf( + param, + "use StarlarkInt, not Integer for parameter '%s' (and see Starlark.toInt)", + paramAnnot.name()); + } + + // Reject an entry of Param.allowedTypes if not assignable to the parameter variable. + for (ParamType paramTypeAnnot : paramAnnot.allowedTypes()) { + TypeMirror t = getParamTypeType(paramTypeAnnot); + if (!types.isAssignable(t, types.erasure(paramType))) { + errorf( + param, + "annotated allowedTypes entry %s of parameter '%s' is not assignable to variable of " + + "type %s", + t, + paramAnnot.name(), + paramType); + } + } + + // Reject generic types C other than C, + // since reflective calls check only the toplevel class. + if (paramType instanceof DeclaredType declaredType) { + for (TypeMirror typeArg : declaredType.getTypeArguments()) { + if (!(typeArg instanceof WildcardType)) { + errorf( + param, + "parameter '%s' has generic type %s, but only wildcard type parameters are" + + " allowed. Type inference in a Starlark-exposed method is unsafe. See" + + " StarlarkMethod class documentation for details.", + param.getSimpleName(), + paramType); + } + } + } + + // Check sense of flag-controlled parameters. + boolean hasFlag = false; + if (!paramAnnot.enableOnlyWithFlag().isEmpty()) { + if (!hasPlusMinusPrefix(paramAnnot.enableOnlyWithFlag())) { + errorf(param, "enableOnlyWithFlag name must have a + or - prefix"); + } + hasFlag = true; + } + if (!paramAnnot.disableWithFlag().isEmpty()) { + if (!hasPlusMinusPrefix(paramAnnot.disableWithFlag())) { + errorf(param, "disableWithFlag name must have a + or - prefix"); + } + if (hasFlag) { + errorf( + param, + "Parameter '%s' has enableOnlyWithFlag and disableWithFlag set. At most one may be set", + paramAnnot.name()); + } + hasFlag = true; + } + if (hasFlag == paramAnnot.valueWhenDisabled().isEmpty()) { + errorf( + param, + hasFlag + ? "Parameter '%s' may be disabled by semantic flag, thus valueWhenDisabled must be" + + " set" + : "Parameter '%s' has valueWhenDisabled set, but is always enabled", + paramAnnot.name()); + } + + // Ensure positional arguments are documented. + if (!paramAnnot.documented() && paramAnnot.positional()) { + errorf( + param, "Parameter '%s' must be documented because it is positional.", paramAnnot.name()); + } + } + + private static boolean hasPlusMinusPrefix(String s) { + return s.charAt(0) == '-' || s.charAt(0) == '+'; + } + + // Returns the logical type of ParamType.type. + private static TypeMirror getParamTypeType(ParamType paramType) { + // See explanation of this hack at Element.getAnnotation + // and at https://stackoverflow.com/a/10167558. + try { + paramType.type(); + throw new IllegalStateException("unreachable"); + } catch (MirroredTypeException ex) { + return ex.getTypeMirror(); + } + } + + private void checkSpecialParams(ExecutableElement method, StarlarkMethod annot) { + if (!annot.extraPositionals().enableOnlyWithFlag().isEmpty() + || !annot.extraPositionals().disableWithFlag().isEmpty()) { + errorf(method, "The extraPositionals parameter may not be toggled by semantic flag"); + } + if (!annot.extraKeywords().enableOnlyWithFlag().isEmpty() + || !annot.extraKeywords().disableWithFlag().isEmpty()) { + errorf(method, "The extraKeywords parameter may not be toggled by semantic flag"); + } + + List params = method.getParameters(); + int index = annot.parameters().length; + + // insufficient parameters? + int special = numExpectedSpecialParams(annot); + if (index + special > params.size()) { + errorf( + method, + "method %s is annotated with %d Params plus %d special parameters, but has only %d" + + " parameter variables", + method.getSimpleName(), + index, + special, + params.size()); + return; // not safe to proceed + } + + if (!annot.extraPositionals().name().isEmpty()) { + VariableElement param = params.get(index++); + // Allow any supertype of Tuple. + TypeMirror tupleType = + types.getDeclaredType(elements.getTypeElement("net.starlark.java.eval.Tuple")); + if (!types.isAssignable(tupleType, param.asType())) { + errorf( + param, + "extraPositionals special parameter '%s' has type %s, to which a Tuple cannot be" + + " assigned", + param.getSimpleName(), + param.asType()); + } + } + + if (!annot.extraKeywords().name().isEmpty()) { + VariableElement param = params.get(index++); + // Allow any supertype of Dict. + TypeMirror dictOfStringObjectType = + types.getDeclaredType( + elements.getTypeElement("net.starlark.java.eval.Dict"), + getType("java.lang.String"), + getType("java.lang.Object")); + if (!types.isAssignable(dictOfStringObjectType, param.asType())) { + errorf( + param, + "extraKeywords special parameter '%s' has type %s, to which Dict" + + " cannot be assigned", + param.getSimpleName(), + param.asType()); + } + } + + if (annot.useStarlarkThread()) { + VariableElement param = params.get(index++); + TypeMirror threadType = getType("net.starlark.java.eval.StarlarkThread"); + if (!types.isSameType(threadType, param.asType())) { + errorf( + param, + "for useStarlarkThread special parameter '%s', got type %s, want StarlarkThread", + param.getSimpleName(), + param.asType()); + } + } + + if (annot.useStarlarkSemantics()) { + VariableElement param = params.get(index++); + TypeMirror semanticsType = getType("net.starlark.java.eval.StarlarkSemantics"); + if (!types.isSameType(semanticsType, param.asType())) { + errorf( + param, + "for useStarlarkSemantics special parameter '%s', got type %s, want StarlarkSemantics", + param.getSimpleName(), + param.asType()); + } + } + + // surplus parameters? + if (index < params.size()) { + errorf( + params.get(index), // first surplus parameter + "method %s is annotated with %d Params plus %d special parameters, yet has %d parameter" + + " variables", + method.getSimpleName(), + annot.parameters().length, + special, + params.size()); + } + } + + private static int numExpectedSpecialParams(StarlarkMethod annot) { + int n = 0; + n += annot.extraPositionals().name().isEmpty() ? 0 : 1; + n += annot.extraKeywords().name().isEmpty() ? 0 : 1; + n += annot.useStarlarkThread() ? 1 : 0; + n += annot.useStarlarkSemantics() ? 1 : 0; + return n; + } + + // Reports a (formatted) error and fails the compilation. + @FormatMethod + private void errorf(Element e, String format, Object... args) { + messager.printMessage(Diagnostic.Kind.ERROR, String.format(format, args), e); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/BUILD b/third_party/bazel/main/java/net/starlark/java/eval/BUILD new file mode 100644 index 000000000..93f62b94c --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/BUILD @@ -0,0 +1,116 @@ +# Description: Bazel's Starlark interpreter + +load("@rules_java//java:defs.bzl", "java_library") + +licenses(["notice"]) # Apache 2.0 + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src/main/java/net/starlark/java:bazel"], +) + +# The Starlark evaluator +java_library( + name = "eval", + srcs = [ + "BuiltinFunction.java", + "CallUtils.java", + "CpuProfiler.java", + "Debug.java", + "Dict.java", + "Eval.java", + "EvalException.java", + "EvalUtils.java", + "FlagGuardedValue.java", + "FormatParser.java", + "GuardedValue.java", + "HasBinary.java", + "ImmutableSingletonStarlarkList.java", + "ImmutableStarlarkList.java", + "JNI.java", + "LazyImmutableStarlarkList.java", + "MethodDescriptor.java", + "MethodLibrary.java", + "Module.java", + "Mutability.java", + "MutableStarlarkList.java", + "NoneType.java", + "ParamDescriptor.java", + "Printer.java", + "RangeList.java", + "RegularImmutableStarlarkList.java", + "RegularTuple.java", + "Sequence.java", + "SingletonTuple.java", + "Starlark.java", + "StarlarkCallable.java", + "StarlarkFloat.java", + "StarlarkFunction.java", + "StarlarkIndexable.java", + "StarlarkInt.java", + "StarlarkIterable.java", + "StarlarkList.java", + "StarlarkMembershipTestable.java", + "StarlarkSemantics.java", + "StarlarkSet.java", + "StarlarkThread.java", + "StarlarkValue.java", + "StringModule.java", + "Structure.java", + "SymbolGenerator.java", + "Tuple.java", + ], + visibility = ["//src/main/java/net/starlark/java:clients"], + deps = [ + # Do not add Bazel or Google dependencies here! + "//src/main/java/net/starlark/java/annot", + "//src/main/java/net/starlark/java/spelling", + "//src/main/java/net/starlark/java/syntax", + "//third_party:error_prone_annotations", + "//third_party:auto_value", + "//third_party:flogger", + "//third_party:guava", + "//third_party:jsr305", + ], +) + +# Dynamic library for Starlark CPU profiler. +# The application or test must arrange for the library to be added +# to the java.library.path directory (e.g. by 'java -Djava.library.path=

'). +# What a mess. +filegroup( + name = "cpu_profiler", + srcs = select({ + "//src/conditions:darwin": [":libcpu_profiler.dylib"], + "//src/conditions:windows": [":cpu_profiler.dll"], + "//conditions:default": [":libcpu_profiler.so"], # POSIX + }), + visibility = ["//src/main/java/net/starlark/java:clients"], +) + +genrule( + name = "cpu_profiler_darwin", + srcs = ["libcpu_profiler.so"], + outs = ["libcpu_profiler.dylib"], + cmd = "cp $< $@", +) + +genrule( + name = "cpu_profiler_windows", + srcs = ["libcpu_profiler.so"], + outs = ["cpu_profiler.dll"], + cmd = "cp $< $@", +) + +# The C++ portion of the Starlark CPU profiler. +cc_binary( + name = "libcpu_profiler.so", + srcs = select({ + "//src/conditions:darwin": ["cpu_profiler_posix.cc"], + "//src/conditions:linux": ["cpu_profiler_posix.cc"], + "//conditions:default": ["cpu_profiler_unimpl.cc"], + }), + linkshared = 1, + deps = ["@bazel_tools//tools/jdk:jni"], +) diff --git a/third_party/bazel/main/java/net/starlark/java/eval/BuiltinFunction.java b/third_party/bazel/main/java/net/starlark/java/eval/BuiltinFunction.java new file mode 100644 index 000000000..c7c4b5eff --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/BuiltinFunction.java @@ -0,0 +1,395 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.eval; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import javax.annotation.Nullable; +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; +import net.starlark.java.spelling.SpellChecker; + +/** + * A BuiltinFunction is a callable Starlark value that reflectively invokes a {@link + * StarlarkMethod}-annotated method of a Java object. The Java object may or may not itself be a + * Starlark value. BuiltinFunctions are not produced for Java methods for which {@link + * StarlarkMethod#structField} is true. + */ +// TODO(adonovan): support annotated static methods. +@StarlarkBuiltin( + name = "builtin_function_or_method", // (following Python) + category = "core", + doc = "The type of a built-in function, defined by Java code.") +public final class BuiltinFunction implements StarlarkCallable { + + private final Object obj; + private final String methodName; + @Nullable private final MethodDescriptor desc; + + /** + * Constructs a BuiltinFunction for a StarlarkMethod-annotated method of the given name (as seen + * by Starlark, not Java). + */ + BuiltinFunction(Object obj, String methodName) { + this.obj = obj; + this.methodName = methodName; + this.desc = null; // computed later + } + + /** + * Constructs a BuiltinFunction for a StarlarkMethod-annotated method (not field) of the given + * name (as seen by Starlark, not Java). + * + *

This constructor should be used only for ephemeral BuiltinFunction values created + * transiently during a call such as {@code x.f()}, when the caller has already looked up the + * MethodDescriptor using the same semantics as the thread that will be used in the call. Use the + * other (slower) constructor if there is any possibility that the semantics of the {@code x.f} + * operation differ from those of the thread used in the call. + */ + BuiltinFunction(Object obj, String methodName, MethodDescriptor desc) { + Preconditions.checkArgument(!desc.isStructField()); + this.obj = obj; + this.methodName = methodName; + this.desc = desc; + } + + @Override + public Object fastcall(StarlarkThread thread, Object[] positional, Object[] named) + throws EvalException, InterruptedException { + MethodDescriptor desc = getMethodDescriptor(thread.getSemantics()); + Object[] vector = getArgumentVector(thread, desc, positional, named); + return desc.call( + obj instanceof String ? StringModule.INSTANCE : obj, vector, thread.mutability()); + } + + private MethodDescriptor getMethodDescriptor(StarlarkSemantics semantics) { + MethodDescriptor desc = this.desc; + if (desc == null) { + desc = CallUtils.getAnnotatedMethods(semantics, obj.getClass()).get(methodName); + Preconditions.checkArgument( + !desc.isStructField(), + "BuiltinFunction constructed for MethodDescriptor(structField=True)"); + } + return desc; + } + + /** + * Returns the StarlarkMethod annotation of this Starlark-callable Java method. + */ + public StarlarkMethod getAnnotation() { + return getMethodDescriptor(StarlarkSemantics.DEFAULT).getAnnotation(); + } + + @Override + public String getName() { + return methodName; + } + + @Override + public void repr(Printer printer) { + if (obj instanceof StarlarkValue || obj instanceof String) { + printer + .append(""); + } else { + printer.append(""); + } + } + + @Override + public String toString() { + return methodName; + } + + /** + * Converts the arguments of a Starlark call into the argument vector for a reflective call to a + * StarlarkMethod-annotated Java method. + * + * @param thread the Starlark thread for the call + * @param loc the location of the call expression, or BUILTIN for calls from Java + * @param desc descriptor for the StarlarkMethod-annotated method + * @param positional an array of positional arguments; as an optimization, in simple cases, this + * array may be reused as the method's return value + * @param named a list of named arguments, as alternating Strings/Objects. May contain dups. + * @return the array of arguments which may be passed to {@link MethodDescriptor#call}. It is + * unsafe to mutate the returned array. + * @throws EvalException if the given set of arguments are invalid for the given method. For + * example, if any arguments are of unexpected type, or not all mandatory parameters are + * specified by the user + */ + private Object[] getArgumentVector( + StarlarkThread thread, + MethodDescriptor desc, // intentionally shadows this.desc + Object[] positional, + Object[] named) + throws EvalException { + + // Overview of steps: + // - allocate vector of actual arguments of correct size. + // - process positional arguments, accumulating surplus ones into *args. + // - process named arguments, accumulating surplus ones into **kwargs. + // - set default values for missing optionals, and report missing mandatory parameters. + // - set special parameters. + // The static checks ensure that positional parameters appear before named, + // and mandatory positionals appear before optional. + // No additional memory allocation occurs in the common (success) case. + // Flag-disabled parameters are skipped during argument matching, as if they do not exist. They + // are instead assigned their flag-disabled values. + + ParamDescriptor[] parameters = desc.getParameters(); + + // Fast case: we only accept positionals and can reuse `positional` as the Java args vector. + // Note that StringModule methods, which are treated specially below, will never match this case + // since their `self` parameter is restricted to String and thus + // isPositionalsReusableAsCallArgsVectorIfArgumentCountValid() would be false. + if (desc.isPositionalsReusableAsJavaArgsVectorIfArgumentCountValid() + && positional.length == parameters.length + && named.length == 0) { + return positional; + } + + // Allocate argument vector. + int n = parameters.length; + if (desc.acceptsExtraArgs()) { + n++; + } + if (desc.acceptsExtraKwargs()) { + n++; + } + if (desc.isUseStarlarkThread()) { + n++; + } + Object[] vector = new Object[n]; + + // positional arguments + int paramIndex = 0; + int argIndex = 0; + if (obj instanceof String) { + // String methods get the string as an extra argument + // because their true receiver is StringModule.INSTANCE. + vector[paramIndex++] = obj; + } + for (; argIndex < positional.length && paramIndex < parameters.length; paramIndex++) { + ParamDescriptor param = parameters[paramIndex]; + if (!param.isPositional()) { + break; + } + + // disabled? + if (param.disabledByFlag() != null) { + // Skip disabled parameter as if not present at all. + // The default value will be filled in below. + continue; + } + + Object value = positional[argIndex++]; + checkParamValue(param, value); + vector[paramIndex] = value; + } + + // *args + Tuple varargs = null; + if (desc.acceptsExtraArgs()) { + varargs = Tuple.wrap(Arrays.copyOfRange(positional, argIndex, positional.length)); + } else if (argIndex < positional.length) { + if (argIndex == 0) { + throw Starlark.errorf("%s() got unexpected positional argument", methodName); + } else { + throw Starlark.errorf( + "%s() accepts no more than %d positional argument%s but got %d", + methodName, argIndex, plural(argIndex), positional.length); + } + } + + // named arguments + LinkedHashMap kwargs = + desc.acceptsExtraKwargs() ? Maps.newLinkedHashMapWithExpectedSize(1) : null; + for (int i = 0; i < named.length; i += 2) { + String name = (String) named[i]; // safe + Object value = named[i + 1]; + + // look up parameter + int index = desc.getParameterIndex(name); + // unknown parameter? + if (index < 0) { + // spill to **kwargs + if (kwargs == null) { + List allNames = + Arrays.stream(parameters) + .map(ParamDescriptor::getName) + .collect(ImmutableList.toImmutableList()); + throw Starlark.errorf( + "%s() got unexpected keyword argument '%s'%s", + methodName, name, SpellChecker.didYouMean(name, allNames)); + } + + // duplicate named argument? + if (kwargs.put(name, value) != null) { + throw Starlark.errorf( + "%s() got multiple values for keyword argument '%s'", methodName, name); + } + continue; + } + ParamDescriptor param = parameters[index]; + + // positional-only param? + if (!param.isNamed()) { + // spill to **kwargs + if (kwargs == null) { + throw Starlark.errorf( + "%s() got named argument for positional-only parameter '%s'", methodName, name); + } + + // duplicate named argument? + if (kwargs.put(name, value) != null) { + throw Starlark.errorf( + "%s() got multiple values for keyword argument '%s'", methodName, name); + } + continue; + } + + // disabled? + String flag = param.disabledByFlag(); + if (flag != null) { + // spill to **kwargs + if (kwargs == null) { + throw Starlark.errorf( + "in call to %s(), parameter '%s' is %s", + methodName, param.getName(), disabled(flag, thread.getSemantics())); + } + + // duplicate named argument? + if (kwargs.put(name, value) != null) { + throw Starlark.errorf( + "%s() got multiple values for keyword argument '%s'", methodName, name); + } + continue; + } + + checkParamValue(param, value); + + // duplicate? + if (vector[index] != null) { + throw Starlark.errorf("%s() got multiple values for argument '%s'", methodName, name); + } + + vector[index] = value; + } + + // Set default values for missing parameters, + // and report any that are still missing. + List missingPositional = null; + List missingNamed = null; + for (int i = 0; i < parameters.length; i++) { + if (vector[i] == null) { + ParamDescriptor param = parameters[i]; + vector[i] = param.getDefaultValue(); + if (vector[i] == null) { + if (param.isPositional()) { + if (missingPositional == null) { + missingPositional = new ArrayList<>(); + } + missingPositional.add(param.getName()); + } else { + if (missingNamed == null) { + missingNamed = new ArrayList<>(); + } + missingNamed.add(param.getName()); + } + } + } + } + if (missingPositional != null) { + throw Starlark.errorf( + "%s() missing %d required positional argument%s: %s", + methodName, + missingPositional.size(), + plural(missingPositional.size()), + Joiner.on(", ").join(missingPositional)); + } + if (missingNamed != null) { + throw Starlark.errorf( + "%s() missing %d required named argument%s: %s", + methodName, + missingNamed.size(), + plural(missingNamed.size()), + Joiner.on(", ").join(missingNamed)); + } + + // special parameters + int i = parameters.length; + if (desc.acceptsExtraArgs()) { + vector[i++] = varargs; + } + if (desc.acceptsExtraKwargs()) { + vector[i++] = Dict.wrap(thread.mutability(), kwargs); + } + if (desc.isUseStarlarkThread()) { + vector[i++] = thread; + } + + return vector; + } + + private static String plural(int n) { + return n == 1 ? "" : "s"; + } + + private void checkParamValue(ParamDescriptor param, Object value) throws EvalException { + List> allowedClasses = param.getAllowedClasses(); + if (allowedClasses == null) { + return; + } + + // Value must belong to one of the specified classes. + boolean ok = false; + for (Class cls : allowedClasses) { + if (cls.isInstance(value)) { + ok = true; + break; + } + } + if (!ok) { + throw Starlark.errorf( + "in call to %s(), parameter '%s' got value of type '%s', want '%s'", + methodName, param.getName(), Starlark.type(value), param.getTypeErrorMessage()); + } + } + + // Returns a phrase meaning "disabled" appropriate to the specified flag. + private static String disabled(String flag, StarlarkSemantics semantics) { + // If the flag is True, it must be a deprecation flag. Otherwise it's an experimental flag. + // TODO(adonovan): is that assumption sound? + if (semantics.getBool(flag)) { + return String.format( + "deprecated and will be removed soon. It may be temporarily re-enabled by setting" + + " --%s=false", + flag.substring(1)); // remove [+-] prefix + } else { + return String.format( + "experimental and thus unavailable with the current flags. It may be enabled by setting" + + " --%s", + flag.substring(1)); // remove [+-] prefix + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/CallUtils.java b/third_party/bazel/main/java/net/starlark/java/eval/CallUtils.java new file mode 100644 index 000000000..1a8805f31 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/CallUtils.java @@ -0,0 +1,234 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nullable; +import net.starlark.java.annot.StarlarkAnnotations; +import net.starlark.java.annot.StarlarkMethod; + +/** Helper functions for {@link StarlarkMethod}-annotated fields and methods. */ +final class CallUtils { + + private CallUtils() {} // uninstantiable + + /** + * Returns the {@link StarlarkClassDescriptor} for the given {@link StarlarkSemantics} and {@link + * Class}. + * + *

This method is a hotspot! It's called on every function call and field access. A single + * `bazel build` invocation can make tens or even hundreds of millions of calls to this method. + */ + private static StarlarkClassDescriptor getStarlarkClassDescriptor( + StarlarkSemantics semantics, Class clazz) { + if (clazz == String.class) { + clazz = StringModule.class; + } + + // We use two layers of caches, with the first layer being keyed by StarlarkSemantics and the + // second layer being keyed by Class. This optimizes for the common case of very few different + // StarlarkSemantics instances (typically, one) being in play. In contrast, if we used a single + // cache data structure then we'd need to use a dedicated tuple object for the keys of that data + // structure, and the GC churn and method call overhead become meaningful at scale. + // + // We implement each cache ourselves using CHM#get and CHM#putIfAbsent. We don't use + // CHM#computeIfAbsent since it is not reentrant: If #getStarlarkClassDescriptor is called + // before Starlark.UNIVERSE is initialized then the computation will re-enter the cache and have + // a cycle; see b/161479826 for history. + // TODO(bazel-team): Maybe the above cycle concern doesn't exist now that CallUtils is private. + ConcurrentHashMap, StarlarkClassDescriptor> starlarkClassDescriptorCache = + starlarkClassDescriptorCachesBySemantics.get( + semantics.getStarlarkClassDescriptorCacheKey()); + if (starlarkClassDescriptorCache == null) { + starlarkClassDescriptorCache = + new ConcurrentHashMap<>( + // In May 2023, typical Bazel usage results in ~150 entries in this cache. Therefore + // we presize the CHM accordingly to reduce the chance two entries use the same hash + // bucket (in May 2023 this strategy was completely effective!). We used to use the + // default capacity, and then the CHM would get dynamically resized to have 256 + // buckets, many of which had at least 2 entries which is suboptimal for such a hot + // data structure. + // TODO(bazel-team): Better would be to precompute the entire lookup table on server + // startup (best would be to do this at compile time via an annotation processor), + // rather than rely on it getting built-up dynamically as Starlark code gets + // evaluated over the lifetime of the server. This way there are no concurrency + // concerns, so we can use a more efficient data structure that doesn't need to + // handle concurrent writes. + /* initialCapacity= */ 1000); + ConcurrentHashMap, StarlarkClassDescriptor> prev = + starlarkClassDescriptorCachesBySemantics.putIfAbsent( + semantics, starlarkClassDescriptorCache); + if (prev != null) { + starlarkClassDescriptorCache = prev; // first thread wins + } + } + + StarlarkClassDescriptor starlarkClassDescriptor = starlarkClassDescriptorCache.get(clazz); + if (starlarkClassDescriptor == null) { + starlarkClassDescriptor = buildStarlarkClassDescriptor(semantics, clazz); + StarlarkClassDescriptor prev = + starlarkClassDescriptorCache.putIfAbsent(clazz, starlarkClassDescriptor); + if (prev != null) { + starlarkClassDescriptor = prev; // first thread wins + } + } + return starlarkClassDescriptor; + } + + /** + * Information derived from a {@link Class} (that has methods annotated with {@link + * StarlarkMethod}) based on a {@link StarlarkSemantics}. + */ + private static class StarlarkClassDescriptor { + @Nullable MethodDescriptor selfCall; + + /** + * All {@link StarlarkMethod}-annotated Java methods, sans ones where {@code selfCall() == + * true}, sorted by Java method name. + */ + ImmutableMap methods; + /** + * Submap of {@link #methods} for which {@code structField() == true}, sorted by Java method + * name. + */ + ImmutableMap fields; + } + + /** + * Two-layer cache of {@link #buildStarlarkClassDescriptor}, managed by {@link + * #getStarlarkClassDescriptor}. + */ + private static final ConcurrentHashMap< + StarlarkSemantics, ConcurrentHashMap, StarlarkClassDescriptor>> + starlarkClassDescriptorCachesBySemantics = new ConcurrentHashMap<>(); + + private static StarlarkClassDescriptor buildStarlarkClassDescriptor( + StarlarkSemantics semantics, Class clazz) { + MethodDescriptor selfCall = null; + ImmutableMap.Builder methods = ImmutableMap.builder(); + Map fields = new HashMap<>(); + + // Sort methods by Java name, for determinism. + Method[] classMethods = clazz.getMethods(); + Arrays.sort(classMethods, Comparator.comparing(Method::getName)); + for (Method method : classMethods) { + // Synthetic methods lead to false multiple matches + if (method.isSynthetic()) { + continue; + } + + // annotated? + StarlarkMethod callable = StarlarkAnnotations.getStarlarkMethod(method); + if (callable == null) { + continue; + } + + // enabled by semantics? + if (!semantics.isFeatureEnabledBasedOnTogglingFlags( + callable.enableOnlyWithFlag(), callable.disableWithFlag())) { + continue; + } + + MethodDescriptor descriptor = MethodDescriptor.of(method, callable, semantics); + + // self-call method? + if (callable.selfCall()) { + if (selfCall != null) { + throw new IllegalArgumentException( + String.format("Class %s has two selfCall methods defined", clazz.getName())); + } + selfCall = descriptor; + continue; + } + + // regular method + methods.put(callable.name(), descriptor); + + // field method? + if (descriptor.isStructField() && fields.put(callable.name(), descriptor) != null) { + // TODO(b/72113542): Validate with annotation processor instead of at runtime. + throw new IllegalArgumentException( + String.format( + "Class %s declares two structField methods named %s", + clazz.getName(), callable.name())); + } + } + + StarlarkClassDescriptor starlarkClassDescriptor = new StarlarkClassDescriptor(); + starlarkClassDescriptor.selfCall = selfCall; + starlarkClassDescriptor.methods = methods.buildOrThrow(); + starlarkClassDescriptor.fields = ImmutableMap.copyOf(fields); + return starlarkClassDescriptor; + } + + /** + * Returns the set of all StarlarkMethod-annotated Java methods (excluding the self-call method) + * of the specified class. + */ + static ImmutableMap getAnnotatedMethods( + StarlarkSemantics semantics, Class objClass) { + return getStarlarkClassDescriptor(semantics, objClass).methods; + } + + /** + * Returns the value of the Starlark field of {@code x}, implemented by a Java method with a + * {@code StarlarkMethod(structField=true)} annotation. + */ + static Object getAnnotatedField(StarlarkSemantics semantics, Object x, String fieldName) + throws EvalException, InterruptedException { + MethodDescriptor desc = + getStarlarkClassDescriptor(semantics, x.getClass()).fields.get(fieldName); + if (desc == null) { + throw Starlark.errorf("value of type %s has no .%s field", Starlark.type(x), fieldName); + } + return desc.callField(x, semantics, /*mu=*/ null); + } + + /** Returns the names of the Starlark fields of {@code x} under the specified semantics. */ + static ImmutableSet getAnnotatedFieldNames(StarlarkSemantics semantics, Object x) { + return getStarlarkClassDescriptor(semantics, x.getClass()).fields.keySet(); + } + + /** + * Returns a {@link MethodDescriptor} object representing a function which calls the selfCall java + * method of the given object (the {@link StarlarkMethod} method with {@link + * StarlarkMethod#selfCall()} set to true). Returns null if no such method exists. + */ + @Nullable + static MethodDescriptor getSelfCallMethodDescriptor( + StarlarkSemantics semantics, Class objClass) { + return getStarlarkClassDescriptor(semantics, objClass).selfCall; + } + + /** + * Returns a {@code selfCall=true} method for the given class under the given Starlark semantics, + * or null if no such method exists. + */ + @Nullable + static Method getSelfCallMethod(StarlarkSemantics semantics, Class objClass) { + MethodDescriptor descriptor = getStarlarkClassDescriptor(semantics, objClass).selfCall; + if (descriptor == null) { + return null; + } + return descriptor.getMethod(); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/CpuProfiler.java b/third_party/bazel/main/java/net/starlark/java/eval/CpuProfiler.java new file mode 100644 index 000000000..acdf80098 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/CpuProfiler.java @@ -0,0 +1,471 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.common.flogger.GoogleLogger; +import java.io.ByteArrayOutputStream; +import java.io.FileDescriptor; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.zip.GZIPOutputStream; +import javax.annotation.Nullable; +import net.starlark.java.syntax.Location; + +// Overview +// +// A CPU profiler measures CPU cycles consumed by each thread. +// It does not account for time a thread is blocked in I/O +// (e.g. within a call to glob), or runnable but not actually +// running, as happens when there are more runnable threads than cores. +// +// CPU profiling requires operating system support. +// On POSIX systems, the setitimer system call causes +// the kernel to signal an application periodically. +// With the ITIMER_PROF option, setitimer delivers a +// SIGPROF signal to a running thread each time its CPU usage +// exceeds the specific quantum. A profiler builds a histogram +// of these these signals, grouped by the current program +// counter location, or more usefully by the complete stack of +// program counter locations. +// +// This profiler calls a C++ function to install a SIGPROF handler. +// Like all handlers for asynchronous signals (that is, signals not +// caused by the execution of program instructions), it is extremely +// constrained in what it may do. It cannot acquire locks, allocate +// memory, or interact with the JVM in any way. Our signal handler +// simply sends a message into a global pipe; the message records +// the operating system's identifier (tid) for the signalled thread. +// +// Reading from the other end of the pipe is a Java thread, the router. +// Its job is to map each OS tid to a StarlarkThread, if the +// thread is currently executing Starlark code, and increment +// a volatile counter in that StarlarkThread. If the thread is +// not executing Starlark code, the router discards the event. +// When a Starlark thread enters or leaves a function during profiling, +// it updates the StarlarkThread-to-OS-thread mapping consulted by the +// router. +// +// If the router does not drain the pipe in a timely manner (on the +// order of 10s; see signal handler), the signal handler prints a +// warning and discards the event. +// +// The router may induce a delay between the kernel signal and the +// thread's stack sampling, during which Starlark execution may have +// moved on to another function. Assuming uniform delay, this is +// equivalent to shifting the phase but not the frequency of CPU ticks. +// Nonetheless it may bias the profile because, for example, +// it would cause a Starlark 'sleep' function to accrue a nonzero +// number of CPU ticks that properly belong to the preceding computation. +// +// When a Starlark thread leaves any function, it reads and clears +// its counter of CPU ticks. If the counter was nonzero, the thread +// writes a copy of its stack to the profiler log in pprof form, +// which is a gzip-compressed stream of protocol messages. +// +// The profiler is inherently global to the process, +// and records the effects of all Starlark threads. +// It may be started and stopped concurrent with Starlark execution, +// allowing profiling of a portion of a long-running computation. + +/** A CPU profiler for Starlark (POSIX only for now). */ +final class CpuProfiler { + + static { + JNI.load(); + } + + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + private final PprofWriter pprof; + + private CpuProfiler(OutputStream out, Duration period) { + this.pprof = new PprofWriter(out, period); + } + + // The active profiler, if any. + @Nullable private static volatile CpuProfiler instance; + + /** Returns the active profiler, or null if inactive. */ + @Nullable + static CpuProfiler get() { + return instance; + } + + // Maps OS thread ID to StarlarkThread. + // The StarlarkThread is needed only for its cpuTicks field. + private static final Map threads = new ConcurrentHashMap<>(); + + /** + * Associates the specified StarlarkThread with the current OS thread. Returns the StarlarkThread + * previously associated with it, if any. + */ + @Nullable + static StarlarkThread setStarlarkThread(StarlarkThread thread) { + if (thread == null) { + return threads.remove(gettid()); + } else { + return threads.put(gettid(), thread); + } + } + + /** Start the profiler. */ + static boolean start(OutputStream out, Duration period) { + if (!supported()) { + logger.atWarning().log("--starlark_cpu_profile is unsupported on this platform"); + return false; + } + if (instance != null) { + throw new IllegalStateException("profiler started twice without intervening stop"); + } + + startRouter(); + if (!startTimer(period.toNanos() / 1000L)) { + throw new IllegalStateException("profile signal handler already in use"); + } + + instance = new CpuProfiler(out, period); + return true; + } + + /** Stop the profiler and wait for the log to be written. */ + static void stop() throws IOException { + if (instance == null) { + throw new IllegalStateException("stop without start"); + } + + CpuProfiler profiler = instance; + instance = null; + + stopTimer(); + + // Finish writing the file and fail if there were any I/O errors. + profiler.pprof.writeEnd(); + } + + /** Records a profile event. */ + void addEvent(int ticks, ImmutableList stack) { + pprof.writeEvent(ticks, stack); + } + + // ---- signal router ---- + + private static FileInputStream pipe; + + // Starts the routing thread if not already started (idempotent). + // On return, it is safe to install the signal handler. + private static synchronized void startRouter() { + if (pipe == null) { + pipe = new FileInputStream(createPipe()); + Thread router = new Thread(CpuProfiler::router, "SIGPROF router"); + router.setDaemon(true); + router.start(); + } + } + + // The Router thread routes SIGPROF events (from the pipe) + // to the relevant StarlarkThread. Once started, it runs forever. + // + // TODO(adonovan): opt: a more efficient implementation of routing would be + // to use, instead of a pipe from the signal handler to the routing thread, + // a mapping, maintained in C++, from OS thread ID to cpuTicks pointer. + // The {add,remove}Thread operations would update this mapping, + // and the signal handler would read it. The mapping would have to + // be a lock-free hash table so that it can be safely read in an + // async signal handler. The pointer would point to the sole element + // of direct memory buffer belonging to the StarlarkThread, allocated + // by JNI NewDirectByteBuffer. + // In this way, the signal handler could update the StarlarkThread directly, + // saving 100 write+read calls per second per core. + // + private static void router() { + byte[] buf = new byte[4]; + while (true) { + try { + int n = pipe.read(buf); + if (n < 0) { + throw new IllegalStateException("pipe closed"); + } + if (n != 4) { + throw new IllegalStateException("short read"); + } + } catch (IOException ex) { + throw new IllegalStateException("unexpected I/O error", ex); + } + + int tid = int32be(buf); + + // Record a CPU tick against tid. + // + // It's not safe to grab the thread's stack here because the thread + // may be changing it, so we increment the thread's counter. + // When the thread later observes the counter is non-zero, + // it gives us the stack by calling addEvent. + StarlarkThread thread = threads.get(tid); + if (thread != null) { + thread.cpuTicks.getAndIncrement(); + } + } + } + + // Decodes a signed 32-bit big-endian integer from b[0:4]. + private static int int32be(byte[] b) { + return b[0] << 24 | (b[1] & 0xff) << 16 | (b[2] & 0xff) << 8 | (b[3] & 0xff); + } + + // --- native code (see cpu_profiler) --- + + // Reports whether the profiler is supported on this platform. + private static native boolean supported(); + + // Returns the read end of a pipe from which profile events may be read. + // Each event is an operating system thread ID encoded as uint32be. + private static native FileDescriptor createPipe(); + + // Starts the operating system's interval timer. + // The period must be a positive number of microseconds. + // Returns false if SIGPROF is already in use. + private static native boolean startTimer(long periodMicros); + + // Stops the operating system's interval timer. + private static native void stopTimer(); + + // Returns the operating system's identifier for the calling thread. + private static native int gettid(); + + // Encoder for pprof format profiles. + // See https://github.com/google/pprof/tree/master/proto + // We encode the protocol messages by hand to avoid + // adding a dependency on the protocol compiler. + private static final class PprofWriter { + + private final Duration period; + private final long startNano; + private GZIPOutputStream gz; + private IOException error; // the first write error, if any; reported during stop() + + PprofWriter(OutputStream out, Duration period) { + this.period = period; + this.startNano = System.nanoTime(); + + try { + this.gz = new GZIPOutputStream(out); + getStringID(""); // entry 0 is always "" + + // dimension and unit + ByteArrayOutputStream unit = new ByteArrayOutputStream(); + writeLong(unit, VALUETYPE_TYPE, getStringID("CPU")); + writeLong(unit, VALUETYPE_UNIT, getStringID("microseconds")); + + // informational fields of Profile + writeByteArray(gz, PROFILE_SAMPLE_TYPE, unit.toByteArray()); + // magnitude of sampling period: + writeLong(gz, PROFILE_PERIOD, period.toNanos() / 1000L); + // dimension and unit of period: + writeByteArray(gz, PROFILE_PERIOD_TYPE, unit.toByteArray()); + // start (real) time of profile: + writeLong(gz, PROFILE_TIME_NANOS, System.currentTimeMillis() * 1000000L); + } catch (IOException ex) { + this.error = ex; + } + } + + synchronized void writeEvent(int ticks, ImmutableList stack) { + if (this.error == null) { + try { + ByteArrayOutputStream sample = new ByteArrayOutputStream(); + writeLong(sample, SAMPLE_VALUE, ticks * period.toNanos() / 1000L); + for (Debug.Frame fr : stack.reverse()) { + writeLong(sample, SAMPLE_LOCATION_ID, getLocationID(fr)); + } + writeByteArray(gz, PROFILE_SAMPLE, sample.toByteArray()); + } catch (IOException ex) { + this.error = ex; + } + } + } + + synchronized void writeEnd() throws IOException { + long endNano = System.nanoTime(); + try { + writeLong(gz, PROFILE_DURATION_NANOS, endNano - startNano); + if (this.error != null) { + throw this.error; // retained from an earlier error + } + } finally { + gz.close(); + } + } + + // Protocol encoding helpers; see https://developers.google.com/protocol-buffers/docs/encoding. + // (Copied to avoid a dependency on the corresponding methods of protobuf.CodedOutputStream.) + + private static void writeLong(OutputStream out, int fieldNumber, long x) throws IOException { + writeVarint(out, (fieldNumber << 3) | 0); // wire type 0 = varint + writeVarint(out, x); + } + + private static void writeString(OutputStream out, int fieldNumber, String x) + throws IOException { + writeByteArray(out, fieldNumber, x.getBytes(UTF_8)); + } + + private static void writeByteArray(OutputStream out, int fieldNumber, byte[] x) + throws IOException { + writeVarint(out, (fieldNumber << 3) | 2); // wire type 2 = length-delimited + writeVarint(out, x.length); + out.write(x); + } + + private static void writeVarint(OutputStream out, long value) throws IOException { + for (; (value & ~0x7f) != 0; value >>>= 7) { + out.write((byte) ((value & 0x7f) | 0x80)); + } + out.write((byte) value); + } + + // Field numbers from pprof protocol. + // See https://github.com/google/pprof/blob/master/proto/profile.proto + private static final int PROFILE_SAMPLE_TYPE = 1; // repeated ValueType + private static final int PROFILE_SAMPLE = 2; // repeated Sample + private static final int PROFILE_MAPPING = 3; // repeated Mapping + private static final int PROFILE_LOCATION = 4; // repeated Location + private static final int PROFILE_FUNCTION = 5; // repeated Function + private static final int PROFILE_STRING_TABLE = 6; // repeated string + private static final int PROFILE_TIME_NANOS = 9; // int64 + private static final int PROFILE_DURATION_NANOS = 10; // int64 + private static final int PROFILE_PERIOD_TYPE = 11; // ValueType + private static final int PROFILE_PERIOD = 12; // int64 + private static final int VALUETYPE_TYPE = 1; // int64 + private static final int VALUETYPE_UNIT = 2; // int64 + private static final int SAMPLE_LOCATION_ID = 1; // repeated uint64 + private static final int SAMPLE_VALUE = 2; // repeated int64 + private static final int SAMPLE_LABEL = 3; // repeated Label + private static final int LABEL_KEY = 1; // int64 + private static final int LABEL_STR = 2; // int64 + private static final int LABEL_NUM = 3; // int64 + private static final int LABEL_NUM_UNIT = 4; // int64 + private static final int LOCATION_ID = 1; // uint64 + private static final int LOCATION_MAPPING_ID = 2; // uint64 + private static final int LOCATION_ADDRESS = 3; // uint64 + private static final int LOCATION_LINE = 4; // repeated Line + private static final int LINE_FUNCTION_ID = 1; // uint64 + private static final int LINE_LINE = 2; // int64 + private static final int FUNCTION_ID = 1; // uint64 + private static final int FUNCTION_NAME = 2; // int64 + private static final int FUNCTION_SYSTEM_NAME = 3; // int64 + private static final int FUNCTION_FILENAME = 4; // int64 + private static final int FUNCTION_START_LINE = 5; // int64 + + // Every string, function, and PC location is emitted once + // and thereafter referred to by its identifier, a Long. + private final Map stringIDs = new HashMap<>(); + private final Map functionIDs = new HashMap<>(); // key is "address" of function + private final Map locationIDs = new HashMap<>(); // key is "address" of PC location + + // Returns the ID of the specified string, + // emitting a pprof string record the first time it is encountered. + private long getStringID(String s) throws IOException { + Long i = stringIDs.putIfAbsent(s, Long.valueOf(stringIDs.size())); + if (i == null) { + writeString(gz, PROFILE_STRING_TABLE, s); + return stringIDs.size() - 1L; + } + return i; + } + + // Returns the ID of a StarlarkCallable for use in Line.FunctionId, + // emitting a pprof Function record the first time fn is encountered. + // The ID is the same as the function's logical address, + // which is supplied by the caller to avoid the need to recompute it. + private long getFunctionID(StarlarkCallable fn, long addr) throws IOException { + Long id = functionIDs.get(addr); + if (id == null) { + id = addr; + + Location loc = fn.getLocation(); + String filename = loc.file(); // TODO(adonovan): make WORKSPACE-relative + String name = fn.getName(); + if (name.equals(StarlarkThread.TOP_LEVEL)) { + name = filename; + } + + long nameID = getStringID(name); + + ByteArrayOutputStream fun = new ByteArrayOutputStream(); + writeLong(fun, FUNCTION_ID, id); + writeLong(fun, FUNCTION_NAME, nameID); + writeLong(fun, FUNCTION_SYSTEM_NAME, nameID); + writeLong(fun, FUNCTION_FILENAME, getStringID(filename)); + writeLong(fun, FUNCTION_START_LINE, (long) loc.line()); + writeByteArray(gz, PROFILE_FUNCTION, fun.toByteArray()); + + functionIDs.put(addr, id); + } + return id; + } + + // Returns the ID of the location denoted by fr, + // emitting a pprof Location record the first time it is encountered. + // For Starlark frames, this is the Frame pc. + private long getLocationID(Debug.Frame fr) throws IOException { + StarlarkCallable fn = fr.getFunction(); + // fnAddr identifies a function as a whole. + int fnAddr = System.identityHashCode(fn); // very imperfect + + // pcAddr identifies the current program point. + // + // For now, this is the same as fnAddr, because + // we don't track the syntax node currently being + // evaluated. Statement-level profile information + // in the leaf function (displayed by 'pprof list ') + // is thus unreliable for now. + long pcAddr = fnAddr; + if (fn instanceof StarlarkFunction) { + // TODO(adonovan): when we use a byte code representation + // of function bodies, mix the program counter fr.pc into fnAddr. + // TODO(adonovan): even cleaner: treat each function's byte + // code segment as its own Profile.Mapping, indexed by pc. + // + // pcAddr = (pcAddr << 16) ^ fr.pc; + } + + Long id = locationIDs.get(pcAddr); + if (id == null) { + id = pcAddr; + + ByteArrayOutputStream line = new ByteArrayOutputStream(); + writeLong(line, LINE_FUNCTION_ID, getFunctionID(fn, fnAddr)); + writeLong(line, LINE_LINE, (long) fr.getLocation().line()); + + ByteArrayOutputStream loc = new ByteArrayOutputStream(); + writeLong(loc, LOCATION_ID, id); + writeLong(loc, LOCATION_ADDRESS, pcAddr); + writeByteArray(loc, LOCATION_LINE, line.toByteArray()); + writeByteArray(gz, PROFILE_LOCATION, loc.toByteArray()); + + locationIDs.put(pcAddr, id); + } + return id; + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Debug.java b/third_party/bazel/main/java/net/starlark/java/eval/Debug.java new file mode 100644 index 000000000..e820b1469 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/Debug.java @@ -0,0 +1,176 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; +import javax.annotation.Nullable; +import net.starlark.java.syntax.Location; + +/** Debugger API. */ +// TODO(adonovan): move Debugger to Debug.Debugger. +public final class Debug { + + /** + * A simple interface for the Starlark interpreter to notify a debugger of events during + * execution. + */ + public interface Debugger { + /** Notify the debugger that execution is at the point immediately before {@code loc}. */ + void before(StarlarkThread thread, Location loc); + + /** Notify the debugger that it will no longer receive events from the interpreter. */ + void close(); + } + + /** A Starlark value that can expose additional information to a debugger. */ + public interface ValueWithDebugAttributes extends StarlarkValue { + /** + * Returns a list of DebugAttribute of this value. For example, it can be the internal fields of + * a value that are not accessible from Starlark, or the values inside a collection. + */ + ImmutableList getDebugAttributes(); + } + + /** A name/value pair used in the return value of getDebugAttributes. */ + public static final class DebugAttribute { + public final String name; + public final Object value; // a legal Starlark value + + public DebugAttribute(String name, Object value) { + this.name = name; + this.value = value; + } + } + + /** See stepControl */ + public interface ReadyToPause extends Predicate {} + + /** + * Describes the stepping behavior that should occur when execution of a thread is continued. + * (Debugger API) + */ + public enum Stepping { + /** Continue execution without stepping. */ + NONE, + /** + * If the thread is paused on a statement that contains a function call, step into that + * function. Otherwise, this is the same as OVER. + */ + INTO, + /** + * Step over the current statement and any functions that it may call, stopping at the next + * statement in the same frame. If no more statements are available in the current frame, same + * as OUT. + */ + OVER, + /** + * Continue execution until the current frame has been exited and then pause. If we are + * currently in the outer-most frame, same as NONE. + */ + OUT, + } + + private Debug() {} // uninstantiable + + static final AtomicReference debugger = new AtomicReference<>(); + + /** + * Installs a global hook that causes subsequently executed Starlark threads to notify the + * debugger of important events. Closes any previously set debugger. Call {@code + * setDebugger(null)} to disable debugging. + */ + public static void setDebugger(Debugger dbg) { + Debugger prev = debugger.getAndSet(dbg); + if (prev != null) { + prev.close(); + } + } + + /** + * Returns a copy of the current stack of call frames, outermost call first. + * + *

This function is intended for use only when execution of {@code thread} is stopped, for + * example at a breakpoint. The resulting DebugFrames should not be retained after execution of + * the thread has resumed. Most clients should instead use {@link StarlarkThread#getCallStack}. + */ + public static ImmutableList getCallStack(StarlarkThread thread) { + return thread.getDebugCallStack(); + } + + /** + * Given a requested stepping behavior, returns a predicate over the context that tells the + * debugger when to pause. (Debugger API) + * + *

The predicate will return true if we are at the next statement where execution should pause, + * and it will return false if we are not yet at that statement. No guarantee is made about the + * predicate's return value after we have reached the desired statement. + * + *

A null return value indicates that no further pausing should occur. + */ + @Nullable + public static Debug.ReadyToPause stepControl(StarlarkThread th, Debug.Stepping stepping) { + final int depth = th.getCallStackSize(); + switch (stepping) { + case NONE: + return null; + case INTO: + // pause at the very next statement + return thread -> true; + case OVER: + return thread -> thread.getCallStackSize() <= depth; + case OUT: + // if we're at the outermost frame, same as NONE + return depth == 0 ? null : thread -> thread.getCallStackSize() < depth; + } + throw new IllegalArgumentException("Unsupported stepping type: " + stepping); + } + + /** Debugger interface to the interpreter's internal call frame representation. */ + public interface Frame { + + /** Returns function called in this frame. */ + StarlarkCallable getFunction(); + + /** Returns the location of the current program counter. */ + Location getLocation(); + + /** Returns the local environment of this frame. */ + ImmutableMap getLocals(); + } + + /** + * Interface by which debugging tools are notified of a thread entering or leaving its top-level + * frame. + */ + public interface ThreadHook { + void onPushFirst(StarlarkThread thread); + + void onPopLast(StarlarkThread thread); + } + + static ThreadHook threadHook = null; + + /** + * Installs a global hook that is notified each time a thread pushes or pops its top-level frame. + * This interface is provided to support special tools; ordinary clients should have no need for + * it. + */ + public static void setThreadHook(ThreadHook hook) { + threadHook = hook; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Dict.java b/third_party/bazel/main/java/net/starlark/java/eval/Dict.java new file mode 100644 index 000000000..376fb140c --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/Dict.java @@ -0,0 +1,818 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import javax.annotation.Nullable; +import net.starlark.java.annot.Param; +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; + +/** + * A Dict is a Starlark dictionary (dict), a mapping from keys to values. + * + *

Dicts are iterable in both Java and Starlark; the iterator yields successive keys. + * + *

Starlark operations on dicts, including element update {@code dict[k]=v} and the {@code + * update} and {@code setdefault} methods, may insert arbitrary Starlark values as dict keys/values, + * regardless of the type argument used to reference the dict from Java code. Therefore, as long as + * a dict is mutable, Java code should refer to it only through a type such as {@code Dict} or {@code Dict} to avoid undermining the type-safety of the Java application. Once + * the dict becomes frozen, it is safe to {@link #cast} it to a more specific type that accurately + * reflects its entries, such as {@code Dict}. + * + *

The following Dict methods, defined by the {@link Map} interface, are not supported. Use the + * corresponding methods with "entry" in their name; they may report mutation failure by throwing a + * checked exception: + * + * 

+ * void clear()         -- use clearEntries
+ * V put(K, V)          -- use putEntry
+ * void putAll(Map)     -- use putEntries
+ * V remove(Object key) -- use removeEntry
+ *
+ */ +@StarlarkBuiltin( + name = "dict", + category = "core", + doc = + "dict is a built-in type representing an associative mapping or dictionary. A" + + " dictionary supports indexing using d[k] and key membership testing" + + " using k in d; both operations take constant time. Unfrozen" + + " dictionaries are mutable, and may be updated by assigning to d[k] or" + + " by calling certain methods. Dictionaries are iterable; iteration yields the" + + " sequence of keys in insertion order. Iteration order is unaffected by updating the" + + " value associated with an existing key, but is affected by removing then reinserting" + + " a key.\n" + + "
d = {0: \"x\", 2: \"z\", 1: \"y\"}\n"
+            + "[k for k in d]  # [0, 2, 1]\n"
+            + "d.pop(2)\n"
+            + "d[0], d[2] = \"a\", \"b\"\n"
+            + "0 in d, \"a\" in d  # (True, False)\n"
+            + "[(k, v) for k, v in d.items()]  # [(0, \"a\"), (1, \"y\"), (2, \"b\")]\n"
+            + "
\n" + + "

There are four ways to construct a dictionary:\n" + + "

    \n" + + "
  1. A dictionary expression {k: v, ...} yields a new dictionary with" + + " the specified key/value entries, inserted in the order they appear in the" + + " expression. Evaluation fails if any two key expressions yield the same value.\n" + + "
  2. A dictionary comprehension {k: v for vars in seq} yields a new" + + " dictionary into which each key/value pair is inserted in loop iteration order." + + " Duplicates are permitted: the first insertion of a given key determines its" + + " position in the sequence, and the last determines its associated value.\n" + + "
    \n"
+            + "{k: v for k, v in ((\"a\", 0), (\"b\", 1), (\"a\", 2))}  # {\"a\": 2, \"b\": 1}\n"
+            + "{i: 2*i for i in range(3)}  # {0: 0, 1: 2, 2: 4}\n"
+            + "
    \n" + + "
  3. A call to the built-in dict function" + + " returns a dictionary containing the specified entries, which are inserted in" + + " argument order, positional arguments before named. As with comprehensions," + + " duplicate keys are permitted.\n" + + "
  4. The union expression x | y yields a new dictionary by combining two" + + " existing dictionaries. If the two dictionaries have a key k in common," + + " the right hand side dictionary's value of the key (in other words," + + " y[k]) wins. The |= variant of the union operator modifies" + + " a dictionary in-place. Example:
    d = {\"foo\":"
+            + " \"FOO\", \"bar\": \"BAR\"} | {\"foo\": \"FOO2\", \"baz\": \"BAZ\"}\n"
+            + "# d == {\"foo\": \"FOO2\", \"bar\": \"BAR\", \"baz\": \"BAZ\"}\n"
+            + "d = {\"a\": 1, \"b\": 2}\n"
+            + "d |= {\"b\": 3, \"c\": 4}\n"
+            + "# d == {\"a\": 1, \"b\": 3, \"c\": 4}
") +public class Dict + implements Map, + StarlarkValue, + Mutability.Freezable, + StarlarkIndexable, + StarlarkIterable { + + private final Map contents; + // Number of active iterators (unused once frozen). + private transient int iteratorCount; // transient for serialization by Bazel + + /** Final except for {@link #unsafeShallowFreeze}; must not be modified any other way. */ + private Mutability mutability; + + private Dict(Mutability mutability, LinkedHashMap contents) { + Preconditions.checkNotNull(mutability); + Preconditions.checkState(mutability != Mutability.IMMUTABLE); + this.mutability = mutability; + // TODO(bazel-team): Memory optimization opportunity: Make it so that a call to + // `mutability.freeze()` causes `contents` here to become an ImmutableMap. Benchmarks show that + // for many targets, this can save a small amount of retained heap (up to 1%). But for some + // targets the bookkeeping required for this causes unacceptably increased temporary heap, and + // the CPU overhead of the bookkeeping and the CPU cost of the ImmutableMap#copyOf call cause + // unacceptably increased CPU. In other words, the overall tradeoff is not obviously worth it + // in all cases. So be careful making this optimization! See comment #12 of b/225469491 for + // details. + this.contents = contents; + } + + private Dict(ImmutableMap contents) { + // An immutable dict might as well store its contents as an ImmutableMap, since ImmutableMap + // both is more memory-efficient than LinkedHashMap and also it has the requisite deterministic + // iteration order. + this.mutability = Mutability.IMMUTABLE; + this.contents = contents; + } + + /** + * Takes ownership of the supplied LinkedHashMap and returns a new Dict that wraps it. The caller + * must not subsequently modify the map, but the Dict may do so. + */ + static Dict wrap(@Nullable Mutability mu, LinkedHashMap contents) { + if (mu == null) { + mu = Mutability.IMMUTABLE; + } + if (mu == Mutability.IMMUTABLE && contents.isEmpty()) { + return empty(); + } + // #wrap is used in situations where the resulting Dict isn't necessarily retained [forever]. + // So, don't make an ImmutableMap copy of `contents`, as #copyOf would do. + return new Dict<>(mu, contents); + } + + @Override + public boolean truth() { + return !isEmpty(); + } + + @Override + public boolean isImmutable() { + return mutability().isFrozen(); + } + + @Override + public boolean updateIteratorCount(int delta) { + if (mutability().isFrozen()) { + return false; + } + if (delta > 0) { + iteratorCount++; + } else if (delta < 0) { + iteratorCount--; + } + return iteratorCount > 0; + } + + @Override + public void checkHashable() throws EvalException { + // Even a frozen dict is unhashable. + throw Starlark.errorf("unhashable type: 'dict'"); + } + + @Override + public int hashCode() { + return contents.hashCode(); + } + + @Override + public boolean equals(Object o) { + return contents.equals(o); + } + + @Override + public Iterator iterator() { + return keySet().iterator(); + } + + @StarlarkMethod( + name = "get", + doc = + "Returns the value for key if key is in the dictionary, " + + "else default. If default is not given, it defaults to " + + "None, so that this method never throws an error.", + parameters = { + @Param(name = "key", doc = "The key to look for."), + @Param( + name = "default", + defaultValue = "None", + named = true, + doc = "The default value to use (instead of None) if the key is not found.") + }, + useStarlarkThread = true) + // TODO(adonovan): This method is named get2 as a temporary workaround for a bug in + // StarlarkAnnotations.getStarlarkMethod. The two 'get' methods cause it to get + // confused as to which one has the annotation. Fix it and remove "2" suffix. + public Object get2(Object key, Object defaultValue, StarlarkThread thread) throws EvalException { + Object v = this.get(key); + if (v != null) { + return v; + } + + // This statement is executed for its effect, which is to throw "unhashable" + // if key is unhashable, instead of returning defaultValue. + // I think this is a bug: the correct behavior is simply 'return defaultValue'. + // See https://github.com/bazelbuild/starlark/issues/65. + containsKey(thread.getSemantics(), key); + + return defaultValue; + } + + @StarlarkMethod( + name = "pop", + doc = + "Removes a key from the dict, and returns the associated value. " + + "If no entry with that key was found, remove nothing and return the specified " + + "default value; if no default value was specified, fail instead.", + parameters = { + @Param(name = "key", doc = "The key."), + @Param( + name = "default", + defaultValue = "unbound", + named = true, + doc = "a default value if the key is absent."), + }, + useStarlarkThread = true) + public Object pop(Object key, Object defaultValue, StarlarkThread thread) throws EvalException { + Starlark.checkMutable(this); + Object value = contents.remove(key); + if (value != null) { + return value; + } + + Starlark.checkHashable(key); + + if (defaultValue != Starlark.UNBOUND) { + return defaultValue; + } + // TODO(adonovan): improve error; this ain't Python. + throw Starlark.errorf("KeyError: %s", Starlark.repr(key)); + } + + @StarlarkMethod( + name = "popitem", + doc = + "Remove and return the first (key, value) pair from the dictionary. " + + "popitem is useful to destructively iterate over a dictionary, " + + "as often used in set algorithms. " + + "If the dictionary is empty, the popitem call fails.") + public Tuple popitem() throws EvalException { + if (isEmpty()) { + throw Starlark.errorf("popitem: empty dictionary"); + } + + Starlark.checkMutable(this); + + Iterator> iterator = contents.entrySet().iterator(); + Entry entry = iterator.next(); + iterator.remove(); + return Tuple.pair(entry.getKey(), entry.getValue()); + } + + @StarlarkMethod( + name = "setdefault", + doc = + "If key is in the dictionary, return its value. " + + "If not, insert key with a value of default " + + "and return default. " + + "default defaults to None.", + parameters = { + @Param(name = "key", doc = "The key."), + @Param( + name = "default", + defaultValue = "None", + named = true, + doc = "a default value if the key is absent."), + }) + public V setdefault(K key, V defaultValue) throws EvalException { + Starlark.checkMutable(this); + Starlark.checkHashable(key); + + V prev = contents.putIfAbsent(key, defaultValue); // see class doc comment + return prev != null ? prev : defaultValue; + } + + @StarlarkMethod( + name = "update", + doc = + "Updates the dictionary first with the optional positional argument, pairs, " + + " then with the optional keyword arguments\n" + + "If the positional argument is present, it must be a dict, iterable, or None.\n" + + "If it is a dict, then its key/value pairs are inserted into this dict. " + + "If it is an iterable, it must provide a sequence of pairs (or other iterables " + + "of length 2), each of which is treated as a key/value pair to be inserted.\n" + + "Each keyword argument name=value causes the name/value " + + "pair to be inserted into this dict.", + parameters = { + @Param( + name = "pairs", + defaultValue = "[]", + doc = + "Either a dictionary or a list of entries. Entries must be tuples or lists with " + + "exactly two elements: key, value."), + }, + extraKeywords = @Param(name = "kwargs", doc = "Dictionary of additional entries."), + useStarlarkThread = true) + public void update(Object pairs, Dict kwargs, StarlarkThread thread) + throws EvalException { + Starlark.checkMutable(this); + @SuppressWarnings("unchecked") + Dict dict = (Dict) this; // see class doc comment + update("update", dict, pairs, kwargs); + } + + // Common implementation of dict(pairs, **kwargs) and dict.update(pairs, **kwargs). + static void update( + String funcname, Dict dict, Object pairs, Map kwargs) + throws EvalException { + if (pairs instanceof Map) { // common case + dict.putEntries((Map) pairs); + } else { + Iterable iterable; + try { + iterable = Starlark.toIterable(pairs); + } catch (EvalException unused) { + throw Starlark.errorf("in %s, got %s, want iterable", funcname, Starlark.type(pairs)); + } + int pos = 0; + for (Object item : iterable) { + Object[] pair; + try { + pair = Starlark.toArray(item); + } catch (EvalException unused) { + throw Starlark.errorf( + "in %s, dictionary update sequence element #%d is not iterable (%s)", + funcname, pos, Starlark.type(item)); + } + if (pair.length != 2) { + throw Starlark.errorf( + "in %s, item #%d has length %d, but exactly two elements are required", + funcname, pos, pair.length); + } + dict.putEntry(pair[0], pair[1]); + pos++; + } + } + + dict.putEntries(kwargs); + } + + @StarlarkMethod( + name = "values", + doc = + "Returns the list of values:" + + "
"
+              + "{2: \"a\", 4: \"b\", 1: \"c\"}.values() == [\"a\", \"b\", \"c\"]
\n", + useStarlarkThread = true) + public StarlarkList values0(StarlarkThread thread) throws EvalException { + return StarlarkList.copyOf(thread.mutability(), values()); + } + + @StarlarkMethod( + name = "items", + doc = + "Returns the list of key-value tuples:" + + "
"
+              + "{2: \"a\", 4: \"b\", 1: \"c\"}.items() == [(2, \"a\"), (4, \"b\"), (1, \"c\")]"
+              + "
\n", + useStarlarkThread = true) + public StarlarkList items(StarlarkThread thread) throws EvalException { + Object[] array = new Object[size()]; + int i = 0; + for (Map.Entry e : contents.entrySet()) { + array[i++] = Tuple.pair(e.getKey(), e.getValue()); + } + return StarlarkList.wrap(thread.mutability(), array); + } + + @StarlarkMethod( + name = "keys", + doc = + "Returns the list of keys:" + + "
{2: \"a\", 4: \"b\", 1: \"c\"}.keys() == [2, 4, 1]"
+              + "
\n", + useStarlarkThread = true) + public StarlarkList keys(StarlarkThread thread) throws EvalException { + Object[] array = new Object[size()]; + int i = 0; + for (K e : contents.keySet()) { + array[i++] = e; + } + return StarlarkList.wrap(thread.mutability(), array); + } + + private static final Dict EMPTY = new Dict<>(ImmutableMap.of()); + + /** Returns an immutable empty dict. */ + // Safe because the empty singleton is immutable. + @SuppressWarnings("unchecked") + public static Dict empty() { + return (Dict) EMPTY; + } + + /** Returns a new empty dict with the specified mutability. */ + public static Dict of(@Nullable Mutability mu) { + if (mu == null) { + mu = Mutability.IMMUTABLE; + } + if (mu == Mutability.IMMUTABLE) { + return empty(); + } else { + return new Dict<>(mu, Maps.newLinkedHashMapWithExpectedSize(1)); + } + } + + /** Returns a new dict with the specified mutability containing the entries of {@code m}. */ + public static Dict copyOf(@Nullable Mutability mu, Map m) { + if (mu == null) { + mu = Mutability.IMMUTABLE; + } + + if (mu == Mutability.IMMUTABLE) { + if (m.isEmpty()) { + return empty(); + } + + if (m instanceof ImmutableMap) { + m.forEach( + (k, v) -> { + Starlark.checkValid(k); + Starlark.checkValid(v); + }); + @SuppressWarnings("unchecked") + ImmutableMap immutableMap = (ImmutableMap) m; + return new Dict<>(immutableMap); + } + + if (m instanceof Dict && ((Dict) m).isImmutable()) { + @SuppressWarnings("unchecked") + Dict dict = (Dict) m; + return dict; + } + + ImmutableMap.Builder immutableMapBuilder = + ImmutableMap.builderWithExpectedSize(m.size()); + m.forEach((k, v) -> immutableMapBuilder.put(Starlark.checkValid(k), Starlark.checkValid(v))); + return new Dict<>(immutableMapBuilder.buildOrThrow()); + } else { + LinkedHashMap linkedHashMap = Maps.newLinkedHashMapWithExpectedSize(m.size()); + m.forEach((k, v) -> linkedHashMap.put(Starlark.checkValid(k), Starlark.checkValid(v))); + return new Dict<>(mu, linkedHashMap); + } + } + + /** Returns an immutable dict containing the entries of {@code m}. */ + public static Dict immutableCopyOf(Map m) { + return copyOf(null, m); + } + + /** Returns a new empty Dict.Builder. */ + public static Builder builder() { + return new Builder<>(); + } + + /** A reusable builder for Dicts. */ + public static final class Builder { + private final ArrayList items = new ArrayList<>(); // [k, v, ... k, v] + + /** Adds an entry (k, v) to the builder, overwriting any previous entry with the same key . */ + @CanIgnoreReturnValue + public Builder put(K k, V v) { + items.add(Starlark.checkValid(k)); + items.add(Starlark.checkValid(v)); + return this; + } + + /** Adds all the map's entries to the builder. */ + @CanIgnoreReturnValue + public Builder putAll(Map map) { + items.ensureCapacity(items.size() + 2 * map.size()); + for (Map.Entry e : map.entrySet()) { + put(e.getKey(), e.getValue()); + } + return this; + } + + /** Returns a new immutable Dict containing the entries added so far. */ + public Dict buildImmutable() { + return build(null); + } + + /** Returns a new {@link ImmutableKeyTrackingDict} containing the entries added so far. */ + public ImmutableKeyTrackingDict buildImmutableWithKeyTracking() { + return new ImmutableKeyTrackingDict<>(buildImmutableMap()); + } + + /** + * Returns a new Dict containing the entries added so far. The result has the specified + * mutability; null means immutable. + */ + public Dict build(@Nullable Mutability mu) { + if (mu == null) { + mu = Mutability.IMMUTABLE; + } + + if (mu == Mutability.IMMUTABLE) { + if (items.isEmpty()) { + return empty(); + } + return new Dict<>(buildImmutableMap()); + } else { + return new Dict<>(mu, buildLinkedHashMap()); + } + } + + private void populateMap(int n, BiConsumer mapEntryConsumer) { + for (int i = 0; i < n; i++) { + @SuppressWarnings("unchecked") + K k = (K) items.get(2 * i); // safe + @SuppressWarnings("unchecked") + V v = (V) items.get(2 * i + 1); // safe + mapEntryConsumer.accept(k, v); + } + } + + private ImmutableMap buildImmutableMap() { + int n = items.size() / 2; + ImmutableMap.Builder immutableMapBuilder = ImmutableMap.builderWithExpectedSize(n); + populateMap(n, immutableMapBuilder::put); + // Respect the desired semantics of Builder#put. + return immutableMapBuilder.buildKeepingLast(); + } + + private LinkedHashMap buildLinkedHashMap() { + int n = items.size() / 2; + LinkedHashMap map = Maps.newLinkedHashMapWithExpectedSize(n); + populateMap(n, map::put); + return map; + } + } + + @Override + public Mutability mutability() { + return mutability; + } + + @Override + public void unsafeShallowFreeze() { + Mutability.Freezable.checkUnsafeShallowFreezePrecondition(this); + this.mutability = Mutability.IMMUTABLE; + } + + /** + * Puts an entry into a dict, after validating that mutation is allowed. + * + * @param key the key of the added entry + * @param value the value of the added entry + * @throws EvalException if the key is invalid or the dict is frozen + */ + public void putEntry(K key, V value) throws EvalException { + Starlark.checkMutable(this); + Starlark.checkHashable(key); + contents.put(key, value); + } + + /** + * Puts all the entries from a given map into the dict, after validating that mutation is allowed. + * + * @param map the map whose entries are added + * @throws EvalException if some key is invalid or the dict is frozen + */ + public void putEntries(Map map) throws EvalException { + Starlark.checkMutable(this); + for (Map.Entry e : map.entrySet()) { + K2 k = e.getKey(); + Starlark.checkHashable(k); + contents.put(k, e.getValue()); + } + } + + /** + * Deletes the entry associated with the given key. + * + * @param key the key to delete + * @return the value associated to the key, or {@code null} if not present + * @throws EvalException if the dict is frozen + */ + V removeEntry(Object key) throws EvalException { + Starlark.checkMutable(this); + return contents.remove(key); + } + + /** + * Clears the dict. + * + * @throws EvalException if the dict is frozen + */ + @StarlarkMethod(name = "clear", doc = "Remove all items from the dictionary.") + public void clearEntries() throws EvalException { + Starlark.checkMutable(this); + contents.clear(); + } + + @Override + public void repr(Printer printer) { + printer.printList(entrySet(), "{", ", ", "}"); + } + + @Override + public String toString() { + return Starlark.repr(this); + } + + /** + * Casts a non-null Starlark value {@code x} to a {@code Dict} after checking that all keys + * and values are instances of {@code keyType} and {@code valueType}, respectively. On error, it + * throws an EvalException whose message includes {@code what}, ideally a string literal, as a + * description of the role of {@code x}. If x is null, it returns an immutable empty dict. + */ + public static Dict cast(Object x, Class keyType, Class valueType, String what) + throws EvalException { + Preconditions.checkNotNull(x); + if (!(x instanceof Dict)) { + throw Starlark.errorf("got %s for '%s', want dict", Starlark.type(x), what); + } + + for (Map.Entry e : ((Map) x).entrySet()) { + if (!keyType.isAssignableFrom(e.getKey().getClass()) + || !valueType.isAssignableFrom(e.getValue().getClass())) { + // TODO(adonovan): change message to "found entry", + // without suggesting that the entire dict is . + throw Starlark.errorf( + "got dict<%s, %s> for '%s', want dict<%s, %s>", + Starlark.type(e.getKey()), + Starlark.type(e.getValue()), + what, + Starlark.classType(keyType), + Starlark.classType(valueType)); + } + } + + @SuppressWarnings("unchecked") // safe + Dict res = (Dict) x; + return res; + } + + /** Like {@link #cast}, but if x is None, returns an empty Dict. */ + public static Dict noneableCast( + Object x, Class keyType, Class valueType, String what) throws EvalException { + return x == Starlark.NONE ? empty() : cast(x, keyType, valueType, what); + } + + @Override + public Object getIndex(StarlarkSemantics semantics, Object key) throws EvalException { + Object v = get(key); + if (v == null) { + throw Starlark.errorf("key %s not found in dictionary", Starlark.repr(key)); + } + return v; + } + + @Override + public boolean containsKey(StarlarkSemantics semantics, Object key) throws EvalException { + Starlark.checkHashable(key); + return this.containsKey(key); + } + + // java.util.Map accessors + + @Override + public boolean containsKey(Object key) { + return contents.containsKey(key); + } + + @Override + public boolean containsValue(Object value) { + return contents.containsValue(value); + } + + @Override + public Set> entrySet() { + return Collections.unmodifiableMap(contents).entrySet(); + } + + @Nullable + @Override + public V get(Object key) { + return contents.get(key); + } + + @Override + public boolean isEmpty() { + return contents.isEmpty(); + } + + @Override + public Set keySet() { + return Collections.unmodifiableMap(contents).keySet(); + } + + @Override + public int size() { + return contents.size(); + } + + @Override + public Collection values() { + return Collections.unmodifiableMap(contents).values(); + } + + // disallowed java.util.Map update operations + + // TODO(adonovan): make mutability exception a subclass of (unchecked) + // UnsupportedOperationException, allowing the primary Dict operations + // to satisfy the Map operations below in the usual way (like ImmutableMap does). + + @Deprecated // use clearEntries + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + @Nullable + @Deprecated // use putEntry + @Override + public V put(K key, V value) { + throw new UnsupportedOperationException(); + } + + @Deprecated // use putEntries + @Override + public void putAll(Map map) { + throw new UnsupportedOperationException(); + } + + @Nullable + @Deprecated // use removeEntry + @Override + public V remove(Object key) { + throw new UnsupportedOperationException(); + } + + /** + * An immutable {@code Dict} that tracks accessed keys. + * + *

Only keys present in the dict are tracked. Any call to {@link #keySet} or {@link #entrySet} + * conservatively results in all keys being considered as accessed - notably, this happens with + * iteration, {@link #repr}, and a mutable copy. + */ + public static final class ImmutableKeyTrackingDict extends Dict { + private final ImmutableSet.Builder accessedKeys = ImmutableSet.builder(); + + private ImmutableKeyTrackingDict(ImmutableMap contents) { + super(contents); + } + + public ImmutableSet getAccessedKeys() { + return accessedKeys.build(); + } + + @Override + @SuppressWarnings("unchecked") // Present keys must be of type K. + public boolean containsKey(Object key) { + if (super.containsKey(key)) { + accessedKeys.add((K) key); + return true; + } + return false; + } + + @Nullable + @Override + @SuppressWarnings("unchecked") // Present keys must be of type K. + public V get(Object key) { + V value = super.get(key); + if (value != null) { + accessedKeys.add((K) key); + } + return value; + } + + @Override + public Set keySet() { + Set keySet = super.keySet(); + accessedKeys.addAll(keySet); + return keySet; + } + + @Override + public Set> entrySet() { + accessedKeys.addAll(super.keySet()); + return super.entrySet(); + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Eval.java b/third_party/bazel/main/java/net/starlark/java/eval/Eval.java new file mode 100644 index 000000000..4bc62e407 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/Eval.java @@ -0,0 +1,899 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import net.starlark.java.spelling.SpellChecker; +import net.starlark.java.syntax.Argument; +import net.starlark.java.syntax.AssignmentStatement; +import net.starlark.java.syntax.BinaryOperatorExpression; +import net.starlark.java.syntax.CallExpression; +import net.starlark.java.syntax.Comprehension; +import net.starlark.java.syntax.ConditionalExpression; +import net.starlark.java.syntax.DefStatement; +import net.starlark.java.syntax.DictExpression; +import net.starlark.java.syntax.DotExpression; +import net.starlark.java.syntax.Expression; +import net.starlark.java.syntax.ExpressionStatement; +import net.starlark.java.syntax.FloatLiteral; +import net.starlark.java.syntax.FlowStatement; +import net.starlark.java.syntax.ForStatement; +import net.starlark.java.syntax.Identifier; +import net.starlark.java.syntax.IfStatement; +import net.starlark.java.syntax.IndexExpression; +import net.starlark.java.syntax.IntLiteral; +import net.starlark.java.syntax.LambdaExpression; +import net.starlark.java.syntax.ListExpression; +import net.starlark.java.syntax.LoadStatement; +import net.starlark.java.syntax.Location; +import net.starlark.java.syntax.Resolver; +import net.starlark.java.syntax.ReturnStatement; +import net.starlark.java.syntax.SliceExpression; +import net.starlark.java.syntax.Statement; +import net.starlark.java.syntax.StringLiteral; +import net.starlark.java.syntax.TokenKind; +import net.starlark.java.syntax.UnaryOperatorExpression; + +final class Eval { + + private Eval() {} // uninstantiable + + // ---- entry point ---- + + // Called from StarlarkFunction.fastcall. + static Object execFunctionBody(StarlarkThread.Frame fr, List statements) + throws EvalException, InterruptedException { + fr.thread.checkInterrupt(); + execStatements(fr, statements, /*indented=*/ false); + return fr.result; + } + + private static StarlarkFunction fn(StarlarkThread.Frame fr) { + return (StarlarkFunction) fr.fn; + } + + private static TokenKind execStatements( + StarlarkThread.Frame fr, List statements, boolean indented) + throws EvalException, InterruptedException { + boolean isToplevelFunction = fn(fr).isToplevel(); + + // Hot code path, good chance of short lists which don't justify the iterator overhead. + for (int i = 0; i < statements.size(); i++) { + Statement stmt = statements.get(i); + TokenKind flow = exec(fr, stmt); + if (flow != TokenKind.PASS) { + return flow; + } + + // Hack for BzlLoadFunction's "export" semantics. + // We enable it only for statements outside any function (isToplevelFunction) + // and outside any if- or for- statements (!indented). + if (isToplevelFunction && !indented && fr.thread.postAssignHook != null) { + if (stmt instanceof AssignmentStatement assign) { + for (Identifier id : Identifier.boundIdentifiers(assign.getLHS())) { + Object value = fn(fr).getGlobal(id.getBinding().getIndex()); + // TODO(bazel-team): Instead of special casing StarlarkFunction, make it implement + // StarlarkExportable. + if (value instanceof StarlarkFunction func) { + // Optimization: The id token of a StarlarkFunction should be based on its global + // identifier when available. This enables an name-based lookup on deserialization. + func.export(fr.thread, id.getName()); + } else { + fr.thread.postAssignHook.assign(id.getName(), value); + } + } + } else if (stmt instanceof DefStatement def) { + Identifier id = def.getIdentifier(); + ((StarlarkFunction) fn(fr).getGlobal(id.getBinding().getIndex())) + .export(fr.thread, id.getName()); + } + } + } + return TokenKind.PASS; + } + + private static void execAssignment(StarlarkThread.Frame fr, AssignmentStatement node) + throws EvalException, InterruptedException { + try { + if (node.isAugmented()) { + execAugmentedAssignment(fr, node); + } else { + Object rvalue = eval(fr, node.getRHS()); + assign(fr, node.getLHS(), rvalue); + } + } catch (EvalException ex) { + fr.setErrorLocation(node.getOperatorLocation()); + throw ex; + } + } + + private static TokenKind execFor(StarlarkThread.Frame fr, ForStatement node) + throws EvalException, InterruptedException { + Iterable seq = evalAsIterable(fr, node.getCollection()); + EvalUtils.addIterator(seq); + try { + for (Object it : seq) { + assign(fr, node.getVars(), it); + + switch (execStatements(fr, node.getBody(), /*indented=*/ true)) { + case PASS: + case CONTINUE: + // Stay in loop. + fr.thread.checkInterrupt(); + continue; + case BREAK: + // Finish loop, execute next statement after loop. + return TokenKind.PASS; + case RETURN: + // Finish loop, return from function. + return TokenKind.RETURN; + default: + throw new IllegalStateException("unreachable"); + } + } + } catch (EvalException ex) { + fr.setErrorLocation(node.getStartLocation()); + throw ex; + } finally { + EvalUtils.removeIterator(seq); + } + return TokenKind.PASS; + } + + private static StarlarkFunction newFunction(StarlarkThread.Frame fr, Resolver.Function rfn) + throws EvalException, InterruptedException { + // Evaluate default value expressions of optional parameters. + // We use MANDATORY to indicate a required parameter + // (not null, because defaults must be a legal tuple value, as + // it will be constructed by the code emitted by the compiler). + // As an optimization, we omit the prefix of MANDATORY parameters. + Object[] defaults = null; + int nparams = + rfn.getParameters().size() - (rfn.hasKwargs() ? 1 : 0) - (rfn.hasVarargs() ? 1 : 0); + for (int i = 0; i < nparams; i++) { + Expression expr = rfn.getParameters().get(i).getDefaultValue(); + if (expr == null && defaults == null) { + continue; // skip prefix of required parameters + } + if (defaults == null) { + defaults = new Object[nparams - i]; + } + defaults[i - (nparams - defaults.length)] = + expr == null ? StarlarkFunction.MANDATORY : eval(fr, expr); + } + if (defaults == null) { + defaults = EMPTY; + } + + // Capture the cells of the function's + // free variables from the lexical environment. + Object[] freevars = new Object[rfn.getFreeVars().size()]; + int i = 0; + for (Resolver.Binding bind : rfn.getFreeVars()) { + // Unlike expr(Identifier), we want the cell itself, not its content. + switch (bind.getScope()) { + case FREE: + freevars[i++] = fn(fr).getFreeVar(bind.getIndex()); + break; + case CELL: + freevars[i++] = fr.locals[bind.getIndex()]; + break; + default: + throw new IllegalStateException("unexpected: " + bind); + } + } + + // Nested functions use the same globalIndex as their enclosing function, + // since both were compiled from the same Program. + StarlarkFunction fn = fn(fr); + return new StarlarkFunction( + rfn, + fn.getModule(), + fn.globalIndex, + Tuple.wrap(defaults), + Tuple.wrap(freevars), + fr.thread.getNextIdentityToken()); + } + + private static TokenKind execIf(StarlarkThread.Frame fr, IfStatement node) + throws EvalException, InterruptedException { + boolean cond = Starlark.truth(eval(fr, node.getCondition())); + if (cond) { + return execStatements(fr, node.getThenBlock(), /*indented=*/ true); + } else if (node.getElseBlock() != null) { + return execStatements(fr, node.getElseBlock(), /*indented=*/ true); + } + return TokenKind.PASS; + } + + private static void execLoad(StarlarkThread.Frame fr, LoadStatement node) throws EvalException { + // Has the application defined a behavior for load statements in this thread? + StarlarkThread.Loader loader = fr.thread.getLoader(); + if (loader == null) { + fr.setErrorLocation(node.getStartLocation()); + throw Starlark.errorf("load statements may not be executed in this thread"); + } + + // Load module. + String moduleName = node.getImport().getValue(); + Module module = loader.load(moduleName); + if (module == null) { + fr.setErrorLocation(node.getStartLocation()); + throw Starlark.errorf("module '%s' not found", moduleName); + } + + for (LoadStatement.Binding binding : node.getBindings()) { + // Extract symbol. + Identifier orig = binding.getOriginalName(); + Object value = module.getGlobal(orig.getName()); + if (value == null) { + fr.setErrorLocation(orig.getStartLocation()); + throw Starlark.errorf( + "file '%s' does not contain symbol '%s'%s", + moduleName, + orig.getName(), + SpellChecker.didYouMean(orig.getName(), module.getGlobals().keySet())); + } + + assignIdentifier(fr, binding.getLocalName(), value); + } + } + + private static TokenKind execReturn(StarlarkThread.Frame fr, ReturnStatement node) + throws EvalException, InterruptedException { + Expression result = node.getResult(); + if (result != null) { + fr.result = eval(fr, result); + } + return TokenKind.RETURN; + } + + private static TokenKind exec(StarlarkThread.Frame fr, Statement st) + throws EvalException, InterruptedException { + if (fr.dbg != null) { + Location loc = st.getStartLocation(); // not very precise + fr.setLocation(loc); + fr.dbg.before(fr.thread, loc); // location is now redundant since it's in the thread + } + + if (++fr.thread.steps >= fr.thread.stepLimit) { + throw new EvalException("Starlark computation cancelled: too many steps"); + } + + switch (st.kind()) { + case ASSIGNMENT: + execAssignment(fr, (AssignmentStatement) st); + return TokenKind.PASS; + case EXPRESSION: + eval(fr, ((ExpressionStatement) st).getExpression()); + return TokenKind.PASS; + case FLOW: + return ((FlowStatement) st).getFlowKind(); + case FOR: + return execFor(fr, (ForStatement) st); + case DEF: + DefStatement def = (DefStatement) st; + StarlarkFunction fn = newFunction(fr, def.getResolvedFunction()); + assignIdentifier(fr, def.getIdentifier(), fn); + return TokenKind.PASS; + case IF: + return execIf(fr, (IfStatement) st); + case LOAD: + execLoad(fr, (LoadStatement) st); + return TokenKind.PASS; + case RETURN: + return execReturn(fr, (ReturnStatement) st); + } + throw new IllegalArgumentException("unexpected statement: " + st.kind()); + } + + /** + * Updates the environment bindings, and possibly mutates objects, so as to assign the given value + * to the given expression. Might not set the frame location on error. + */ + private static void assign(StarlarkThread.Frame fr, Expression lhs, Object value) + throws EvalException, InterruptedException { + if (lhs instanceof Identifier ident) { + // x = ... + assignIdentifier(fr, ident, value); + + } else if (lhs instanceof IndexExpression index) { + // x[i] = ... + Object object = eval(fr, index.getObject()); + Object key = eval(fr, index.getKey()); + EvalUtils.setIndex(object, key, value); + + } else if (lhs instanceof ListExpression list) { + // a, b, c = ... + assignSequence(fr, list.getElements(), value); + + } else if (lhs instanceof DotExpression dot) { + // x.f = ... + Object object = eval(fr, dot.getObject()); + String field = dot.getField().getName(); + try { + EvalUtils.setField(object, field, value); + } catch (EvalException ex) { + fr.setErrorLocation(dot.getDotLocation()); + throw ex; + } + } else { + // Not possible for resolved ASTs. + throw Starlark.errorf("cannot assign to '%s'", lhs); + } + } + + private static void assignIdentifier(StarlarkThread.Frame fr, Identifier id, Object value) + throws EvalException { + Resolver.Binding bind = id.getBinding(); + switch (bind.getScope()) { + case LOCAL: + fr.locals[bind.getIndex()] = value; + break; + case CELL: + ((StarlarkFunction.Cell) fr.locals[bind.getIndex()]).x = value; + break; + case GLOBAL: + fn(fr).setGlobal(bind.getIndex(), value); + break; + default: + throw new IllegalStateException(bind.getScope().toString()); + } + } + + /** + * Recursively assigns an iterable value to a non-empty sequence of assignable expressions. Might + * not set frame location on error. + */ + private static void assignSequence(StarlarkThread.Frame fr, List lhs, Object x) + throws EvalException, InterruptedException { + // TODO(adonovan): lock/unlock rhs during iteration so that + // assignments fail when the left side aliases the right, + // which is a tricky case in Python assignment semantics. + int nrhs = Starlark.len(x); + int nlhs = lhs.size(); + if (nrhs < 0 || x instanceof String) { // strings are not iterable + throw Starlark.errorf( + "got '%s' in sequence assignment (want %d-element sequence)", Starlark.type(x), nlhs); + } + Iterable rhs = Starlark.toIterable(x); + if (nlhs != nrhs) { + throw Starlark.errorf( + "too %s values to unpack (got %d, want %d)", nrhs < nlhs ? "few" : "many", nrhs, nlhs); + } + int i = 0; + for (Object item : rhs) { + assign(fr, lhs.get(i), item); + i++; + } + } + + // Might not set frame location on error. + private static void execAugmentedAssignment(StarlarkThread.Frame fr, AssignmentStatement stmt) + throws EvalException, InterruptedException { + Expression lhs = stmt.getLHS(); + TokenKind op = stmt.getOperator(); + Expression rhs = stmt.getRHS(); + + if (lhs instanceof Identifier ident) { + // x op= y (lhs must be evaluated only once) + Object x = eval(fr, lhs); + Object y = eval(fr, rhs); + Object z; + try { + z = inplaceBinaryOp(fr, op, x, y); + } catch (EvalException ex) { + fr.setErrorLocation(stmt.getOperatorLocation()); + throw ex; + } + assignIdentifier(fr, ident, z); + + } else if (lhs instanceof IndexExpression index) { + // object[index] op= y + // The object and key should be evaluated only once, so we don't use lhs.eval(). + Object object = eval(fr, index.getObject()); + Object key = eval(fr, index.getKey()); + Object x = EvalUtils.index(fr.thread, object, key); + // Evaluate rhs after lhs. + Object y = eval(fr, rhs); + Object z; + try { + z = inplaceBinaryOp(fr, op, x, y); + } catch (EvalException ex) { + fr.setErrorLocation(stmt.getOperatorLocation()); + throw ex; + } + try { + EvalUtils.setIndex(object, key, z); + } catch (EvalException ex) { + fr.setErrorLocation(stmt.getOperatorLocation()); + throw ex; + } + + } else if (lhs instanceof DotExpression dot) { + // object.field op= y (lhs must be evaluated only once) + Object object = eval(fr, dot.getObject()); + String field = dot.getField().getName(); + try { + Object x = + Starlark.getattr( + fr.thread.mutability(), + fr.thread.getSemantics(), + object, + field, + /*defaultValue=*/ null); + Object y = eval(fr, rhs); + Object z; + try { + z = inplaceBinaryOp(fr, op, x, y); + } catch (EvalException ex) { + fr.setErrorLocation(stmt.getOperatorLocation()); + throw ex; + } + EvalUtils.setField(object, field, z); + } catch (EvalException ex) { + fr.setErrorLocation(dot.getDotLocation()); + throw ex; + } + + } else { + // Not possible for resolved ASTs. + fr.setErrorLocation(stmt.getOperatorLocation()); + throw Starlark.errorf("cannot perform augmented assignment on '%s'", lhs); + } + } + + private static Object inplaceBinaryOp(StarlarkThread.Frame fr, TokenKind op, Object x, Object y) + throws EvalException { + switch (op) { + case PLUS: + // list += iterable behaves like list.extend(iterable) + // TODO(b/141263526): following Python, allow list+=iterable (but not list+iterable). + if (x instanceof StarlarkList xList && y instanceof StarlarkList yList) { + xList.extend(yList); + return xList; + } + break; + + case PIPE: + if (x instanceof Dict && y instanceof Map) { + // dict |= map merges the contents of the second operand (usually a dict) into the first. + @SuppressWarnings("unchecked") + Dict xDict = (Dict) x; + @SuppressWarnings("unchecked") + Map yMap = (Map) y; + xDict.putEntries(yMap); + return xDict; + } else if (x instanceof StarlarkSet xSet && y instanceof Set ySet) { + // set |= set merges the contents of the second operand into the first. + xSet.update(Tuple.of(ySet)); + return xSet; + } + break; + + case AMPERSAND: + if (x instanceof StarlarkSet xSet && y instanceof Set ySet) { + // set &= set replaces the first set with the intersection of the two sets. + xSet.intersectionUpdate(Tuple.of(ySet)); + return xSet; + } + break; + + case CARET: + if (x instanceof StarlarkSet xSet && y instanceof Set ySet) { + // set ^= set replaces the first set with the symmetric difference of the two sets. + xSet.symmetricDifferenceUpdate(ySet); + return xSet; + } + break; + + case MINUS: + if (x instanceof StarlarkSet xSet && y instanceof Set ySet) { + // set -= set removes all elements of the second set from the first set. + xSet.differenceUpdate(Tuple.of(ySet)); + return xSet; + } + break; + + default: // fall through + } + return EvalUtils.binaryOp(op, x, y, fr.thread); + } + + // ---- expressions ---- + + private static Object eval(StarlarkThread.Frame fr, Expression expr) + throws EvalException, InterruptedException { + if (++fr.thread.steps >= fr.thread.stepLimit) { + throw new EvalException("Starlark computation cancelled: too many steps"); + } + + // The switch cases have been split into separate functions + // to reduce the stack usage during recursion, which is + // especially important in practice for deeply nested a+...+z + // expressions; see b/153764542. + switch (expr.kind()) { + case BINARY_OPERATOR: + return evalBinaryOperator(fr, (BinaryOperatorExpression) expr); + case COMPREHENSION: + return evalComprehension(fr, (Comprehension) expr); + case CONDITIONAL: + return evalConditional(fr, (ConditionalExpression) expr); + case DICT_EXPR: + return evalDict(fr, (DictExpression) expr); + case DOT: + return evalDot(fr, (DotExpression) expr); + case CALL: + return evalCall(fr, (CallExpression) expr); + case IDENTIFIER: + return evalIdentifier(fr, (Identifier) expr); + case INDEX: + return evalIndex(fr, (IndexExpression) expr); + case INT_LITERAL: + // TODO(adonovan): opt: avoid allocation by saving + // the StarlarkInt in the IntLiteral (a temporary hack + // until we use a compiled representation). + Number n = ((IntLiteral) expr).getValue(); + if (n instanceof Integer nInt) { + return StarlarkInt.of(nInt); + } else if (n instanceof Long nLong) { + return StarlarkInt.of(nLong); + } else { + return StarlarkInt.of((BigInteger) n); + } + case FLOAT_LITERAL: + return StarlarkFloat.of(((FloatLiteral) expr).getValue()); + case LAMBDA: + return newFunction(fr, ((LambdaExpression) expr).getResolvedFunction()); + case LIST_EXPR: + return evalList(fr, (ListExpression) expr); + case SLICE: + return evalSlice(fr, (SliceExpression) expr); + case STRING_LITERAL: + return ((StringLiteral) expr).getValue(); + case UNARY_OPERATOR: + return evalUnaryOperator(fr, (UnaryOperatorExpression) expr); + } + throw new IllegalArgumentException("unexpected expression: " + expr.kind()); + } + + private static Object evalBinaryOperator(StarlarkThread.Frame fr, BinaryOperatorExpression binop) + throws EvalException, InterruptedException { + Object x = eval(fr, binop.getX()); + // AND and OR require short-circuit evaluation. + switch (binop.getOperator()) { + case AND: + return Starlark.truth(x) ? eval(fr, binop.getY()) : x; + case OR: + return Starlark.truth(x) ? x : eval(fr, binop.getY()); + default: + Object y = eval(fr, binop.getY()); + try { + return EvalUtils.binaryOp(binop.getOperator(), x, y, fr.thread); + } catch (EvalException ex) { + fr.setErrorLocation(binop.getOperatorLocation()); + throw ex; + } + } + } + + private static Object evalConditional(StarlarkThread.Frame fr, ConditionalExpression cond) + throws EvalException, InterruptedException { + Object v = eval(fr, cond.getCondition()); + return eval(fr, Starlark.truth(v) ? cond.getThenCase() : cond.getElseCase()); + } + + private static Object evalDict(StarlarkThread.Frame fr, DictExpression dictexpr) + throws EvalException, InterruptedException { + Dict dict = Dict.of(fr.thread.mutability()); + for (DictExpression.Entry entry : dictexpr.getEntries()) { + Object k = eval(fr, entry.getKey()); + Object v = eval(fr, entry.getValue()); + int before = dict.size(); + try { + dict.putEntry(k, v); + } catch (EvalException ex) { + fr.setErrorLocation(entry.getColonLocation()); + throw ex; + } + if (dict.size() == before) { + fr.setErrorLocation(entry.getColonLocation()); + throw Starlark.errorf("dictionary expression has duplicate key: %s", Starlark.repr(k)); + } + } + return dict; + } + + private static Object evalDot(StarlarkThread.Frame fr, DotExpression dot) + throws EvalException, InterruptedException { + Object object = eval(fr, dot.getObject()); + String name = dot.getField().getName(); + try { + return Starlark.getattr( + fr.thread.mutability(), fr.thread.getSemantics(), object, name, /*defaultValue=*/ null); + } catch (EvalException ex) { + fr.setErrorLocation(dot.getDotLocation()); + throw ex; + } + } + + private static Object evalCall(StarlarkThread.Frame fr, CallExpression call) + throws EvalException, InterruptedException { + fr.thread.checkInterrupt(); + + Object fn = eval(fr, call.getFunction()); + + // Starlark arguments are ordered: positionals < keywords < *args < **kwargs. + // + // This is stricter than Python2, which doesn't constrain keywords wrt *args, + // but this ensures that the effects of evaluation of Starlark arguments occur + // in source order. + // + // Starlark does not support Python3's multiple *args and **kwargs + // nor freer ordering, such as f(a, *list, *list, **dict, **dict, b=1). + // Supporting it would complicate a compiler, and produce effects out of order. + // Also, Python's argument ordering rules are complex and the errors sometimes cryptic. + + // StarStar and Star args are guaranteed to be last, if they occur. + ImmutableList arguments = call.getArguments(); + int n = arguments.size(); + Argument.StarStar starstar = null; + if (n > 0 && arguments.get(n - 1) instanceof Argument.StarStar) { + starstar = (Argument.StarStar) arguments.get(n - 1); + n--; + } + Argument.Star star = null; + if (n > 0 && arguments.get(n - 1) instanceof Argument.Star) { + star = (Argument.Star) arguments.get(n - 1); + n--; + } + // Inv: n = |positional| + |named| + + // Allocate assuming no *args/**kwargs. + int npos = call.getNumPositionalArguments(); + int i; + + // f(expr) -- positional args + Object[] positional = npos == 0 ? EMPTY : new Object[npos]; + for (i = 0; i < npos; i++) { + Argument arg = arguments.get(i); + Object value = eval(fr, arg.getValue()); + positional[i] = value; + } + + // f(id=expr) -- named args + Object[] named = n == npos ? EMPTY : new Object[2 * (n - npos)]; + for (int j = 0; i < n; i++) { + Argument.Keyword arg = (Argument.Keyword) arguments.get(i); + Object value = eval(fr, arg.getValue()); + named[j++] = arg.getName(); + named[j++] = value; + } + + // f(*args) -- varargs + if (star != null) { + Object value = eval(fr, star.getValue()); + if (!(value instanceof StarlarkIterable iter)) { + fr.setErrorLocation(star.getStartLocation()); + throw Starlark.errorf("argument after * must be an iterable, not %s", Starlark.type(value)); + } + // TODO(adonovan): opt: if value.size is known, preallocate (and skip if empty). + ArrayList list = new ArrayList<>(); + Collections.addAll(list, positional); + Iterables.addAll(list, iter); + positional = list.toArray(); + } + + // f(**kwargs) + if (starstar != null) { + Object value = eval(fr, starstar.getValue()); + // Unlike *args, we don't have a Starlark-specific mapping interface to check for in **kwargs, + // so check for Java's Map instead. + if (!(value instanceof Map kwargs)) { + fr.setErrorLocation(starstar.getStartLocation()); + throw Starlark.errorf("argument after ** must be a dict, not %s", Starlark.type(value)); + } + int j = named.length; + named = Arrays.copyOf(named, j + 2 * kwargs.size()); + for (Map.Entry e : kwargs.entrySet()) { + if (!(e.getKey() instanceof String)) { + fr.setErrorLocation(starstar.getStartLocation()); + throw Starlark.errorf("keywords must be strings, not %s", Starlark.type(e.getKey())); + } + named[j++] = e.getKey(); + named[j++] = e.getValue(); + } + } + + Location loc = call.getLparenLocation(); // (Location is prematerialized) + fr.setLocation(loc); + try { + return Starlark.fastcall(fr.thread, fn, positional, named); + } catch (EvalException ex) { + fr.setErrorLocation(loc); + throw ex; + } + } + + private static Object evalIdentifier(StarlarkThread.Frame fr, Identifier id) + throws EvalException, InterruptedException { + Resolver.Binding bind = id.getBinding(); + Object result; + switch (bind.getScope()) { + case LOCAL: + result = fr.locals[bind.getIndex()]; + break; + case CELL: + result = ((StarlarkFunction.Cell) fr.locals[bind.getIndex()]).x; + break; + case FREE: + result = fn(fr).getFreeVar(bind.getIndex()).x; + break; + case GLOBAL: + result = fn(fr).getGlobal(bind.getIndex()); + break; + case PREDECLARED: + result = fn(fr).getModule().getPredeclared(id.getName()); + break; + case UNIVERSAL: + result = Starlark.UNIVERSE.get(id.getName()); + break; + default: + throw new IllegalStateException(bind.toString()); + } + if (result == null) { + fr.setErrorLocation(id.getStartLocation()); + throw Starlark.errorf( + "%s variable '%s' is referenced before assignment.", bind.getScope(), id.getName()); + } + return result; + } + + private static Object evalIndex(StarlarkThread.Frame fr, IndexExpression index) + throws EvalException, InterruptedException { + Object object = eval(fr, index.getObject()); + Object key = eval(fr, index.getKey()); + try { + return EvalUtils.index(fr.thread, object, key); + } catch (EvalException ex) { + fr.setErrorLocation(index.getLbracketLocation()); + throw ex; + } + } + + private static Object evalList(StarlarkThread.Frame fr, ListExpression expr) + throws EvalException, InterruptedException { + int n = expr.getElements().size(); + Object[] array = new Object[n]; + for (int i = 0; i < n; i++) { + array[i] = eval(fr, expr.getElements().get(i)); + } + return expr.isTuple() ? Tuple.wrap(array) : StarlarkList.wrap(fr.thread.mutability(), array); + } + + private static Object evalSlice(StarlarkThread.Frame fr, SliceExpression slice) + throws EvalException, InterruptedException { + Object x = eval(fr, slice.getObject()); + Object start = slice.getStart() == null ? Starlark.NONE : eval(fr, slice.getStart()); + Object stop = slice.getStop() == null ? Starlark.NONE : eval(fr, slice.getStop()); + Object step = slice.getStep() == null ? Starlark.NONE : eval(fr, slice.getStep()); + try { + return Starlark.slice(fr.thread.mutability(), x, start, stop, step); + } catch (EvalException ex) { + fr.setErrorLocation(slice.getLbracketLocation()); + throw ex; + } + } + + private static Object evalUnaryOperator(StarlarkThread.Frame fr, UnaryOperatorExpression unop) + throws EvalException, InterruptedException { + Object x = eval(fr, unop.getX()); + try { + return EvalUtils.unaryOp(unop.getOperator(), x); + } catch (EvalException ex) { + fr.setErrorLocation(unop.getStartLocation()); + throw ex; + } + } + + private static Object evalComprehension(StarlarkThread.Frame fr, Comprehension comp) + throws EvalException, InterruptedException { + final Dict dict = comp.isDict() ? Dict.of(fr.thread.mutability()) : null; + final StarlarkList list = + comp.isDict() ? null : StarlarkList.newList(fr.thread.mutability()); + + // The Lambda class serves as a recursive lambda closure. + class Lambda { + // execClauses(index) recursively executes the clauses starting at index, + // and finally evaluates the body and adds its value to the result. + void execClauses(int index) throws EvalException, InterruptedException { + fr.thread.checkInterrupt(); + + // recursive case: one or more clauses + if (index < comp.getClauses().size()) { + Comprehension.Clause clause = comp.getClauses().get(index); + if (clause instanceof Comprehension.For forClause) { + + Iterable seq = evalAsIterable(fr, forClause.getIterable()); + EvalUtils.addIterator(seq); + try { + for (Object elem : seq) { + assign(fr, forClause.getVars(), elem); + execClauses(index + 1); + } + } catch (EvalException ex) { + fr.setErrorLocation(forClause.getStartLocation()); + throw ex; + } finally { + EvalUtils.removeIterator(seq); + } + + } else { + Comprehension.If ifClause = (Comprehension.If) clause; + if (Starlark.truth(eval(fr, ifClause.getCondition()))) { + execClauses(index + 1); + } + } + return; + } + + // base case: evaluate body and add to result. + if (dict != null) { + DictExpression.Entry body = (DictExpression.Entry) comp.getBody(); + Object k = eval(fr, body.getKey()); + try { + Starlark.checkHashable(k); + Object v = eval(fr, body.getValue()); + dict.putEntry(k, v); + } catch (EvalException ex) { + fr.setErrorLocation(body.getColonLocation()); + throw ex; + } + } else { + list.addElement(eval(fr, ((Expression) comp.getBody()))); + } + } + } + new Lambda().execClauses(0); + + return comp.isDict() ? dict : list; + } + + /** + * Evaluates an expression to an iterable Starlark value and returns an {@code Iterable} view of + * it. If evaluation fails or the value is not iterable, throws {@code EvalException} and sets the + * error location to the expression's start. + */ + private static Iterable evalAsIterable(StarlarkThread.Frame fr, Expression expr) + throws EvalException, InterruptedException { + Object o = eval(fr, expr); + try { + return Starlark.toIterable(o); + } catch (EvalException ex) { + fr.setErrorLocation(expr.getStartLocation()); + throw ex; + } + } + + private static final Object[] EMPTY = {}; +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/EvalException.java b/third_party/bazel/main/java/net/starlark/java/eval/EvalException.java new file mode 100644 index 000000000..c8df11595 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/EvalException.java @@ -0,0 +1,230 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.io.Files; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.io.File; +import java.util.List; +import java.util.function.Supplier; +import javax.annotation.Nullable; +import net.starlark.java.syntax.Location; + +/** An EvalException indicates an Starlark evaluation error. */ +public class EvalException extends Exception { + + // The call stack associated with this error. + // It is initially null, but is set by the interpreter to a non-empty + // stack when popping a frame. Thus an exception newly created by a + // built-in function has no stack until it is thrown out of a function call. + @Nullable private ImmutableList callstack; + + /** Constructs an EvalException. Use {@link Starlark#errorf} if you want string formatting. */ + public EvalException(String message) { + this(message, /* cause= */ (Throwable) null); + } + + /** + * Constructs an EvalException with a message and optional cause. + * + *

The cause does not affect the error message, so callers should incorporate {@code + * cause.getMessage()} into {@code message} if desired, or call {@code EvalException(Throwable)}. + */ + public EvalException(String message, @Nullable Throwable cause) { + super(Preconditions.checkNotNull(message), cause); + } + + /** Constructs an EvalException using the same message as the cause exception. */ + public EvalException(Throwable cause) { + super(getCauseMessage(cause), cause); + } + + /** Fills in the callstack if it hasn't been set yet. */ + @CanIgnoreReturnValue + public EvalException withCallStack(List callstack) { + if (this.callstack == null) { + this.callstack = ImmutableList.copyOf(callstack); + } + return this; + } + + private static String getCauseMessage(Throwable cause) { + String msg = cause.getMessage(); + return msg != null ? msg : cause.toString(); + } + + /** Returns the error message. Does not include call stack or cause. */ + @Override + public final String getMessage() { + return super.getMessage(); + } + + /** + * Returns the call stack associated with this error, outermost call first. A newly constructed + * exception has an empty stack, but an exception that has been thrown out of a Starlark function + * call has its stack populated automatically. The identity of the thrown exception does not + * change. + * + *

EvalException is widely used to indicate the failure of basic operations on Starlark values, + * such as those corresponding to the Starlark expressions {@code x.f}, {@code x[i]}, {@code x+y}, + * and so on, even when these failing operations occur outside the context of a StarlarkThread or + * the interpreter. EvalExceptions from such failures do not have an associated stack. + * + *

For best results, when handling an EvalException, print the stack, using {@link + * #getMessageWithStack} to display multiple complete lines of output, only if the exception + * resulted from Starlark evaluation. For an EvalException with no stack, use {@link #getMessage} + * to obtain a message suitable for incorporating into a larger error. + */ + public final ImmutableList getCallStack() { + return callstack != null ? callstack : ImmutableList.of(); + } + + /** Returns the innermost non-builtin location in the call stack, or null if there is none. */ + @Nullable + public Location getInnermostLocation() { + if (callstack == null) { + return null; + } + return callstack.reverse().stream() + .map(entry -> entry.location) + .filter(location -> location != Location.BUILTIN) + .findFirst() + .orElse(null); + } + + /** Returns the error message along with its call stack. May be overridden by subclasses. */ + @Override + public String toString() { + return getMessageWithStack(); + } + + /** + * Returns the error message along with its call stack, if any. Equivalent to {@code + * getMessageWithStack(newSourceReader())}. + */ + public final String getMessageWithStack() { + return getMessageWithStack(newSourceReader()); + } + + /** + * Returns the error message along with its call stack, if any (see {@link #getCallStack}). The + * source line for each stack frame is obtained from the provided SourceReader. + */ + public final String getMessageWithStack(SourceReader src) { + if (callstack != null) { + return formatCallStack(callstack, getMessage(), src); + } + return getMessage(); + } + + /** + * A SourceReader reads the line of source denoted by a Location to be displayed in a formatted + * stack trace. + */ + public interface SourceReader { + /** Returns a single line of source code (sans newline), or null if unavailable. */ + String readline(Location loc); + } + + /** + * Sets the function used to obtain a SourceReader when subsequently formatting a call stack. + * + *

The default supplier returns SourceReaders that read from the file system, but a + * security-conscious client may wish to disable this capability or provide an alternative. + */ + public static synchronized void setSourceReaderSupplier(Supplier f) { + sourceReaderSupplier = f; + } + + /** Returns a new SourceReader. See {@link #setSourceReaderSupplier}. */ + public static synchronized SourceReader newSourceReader() { + return sourceReaderSupplier.get(); + } + + private static Supplier sourceReaderSupplier = + () -> { + // TODO(adonovan): opt: cache seen files, as the stack often repeats the same files. + return loc -> { + try { + String content = Files.asCharSource(new File(loc.file()), UTF_8).read(); + return Iterables.get(Splitter.on("\n").split(content), loc.line() - 1, null); + } catch (Throwable unused) { + // ignore any failure (e.g. security manager rejecting I/O) + } + return null; + }; + }; + + /** + * Formats the given call stack and error message. Provided as a separate function from {@link + * #getMessageWithStack} so that clients may modify the stack and/or error before formatting it. + * The source line for each stack frame is obtained from the provided SourceReader. + */ + public static String formatCallStack( + List callstack, String message, SourceReader src) { + StringBuilder buf = new StringBuilder(); + int n = callstack.size(); // n > 0 + String prefix = "Error: "; + // If the topmost frame is a built-in, don't show it. + // Instead just prefix the name of the built-in onto the error message. + StarlarkThread.CallStackEntry leaf = callstack.get(n - 1); + if (leaf.location.equals(Location.BUILTIN)) { + prefix = "Error in " + leaf.name + ": "; + n--; + } + if (n > 0) { + buf.append("Traceback (most recent call last):\n"); + for (int i = 0; i < n; i++) { + StarlarkThread.CallStackEntry fr = callstack.get(i); + // 'File "file.bzl", line 1, column 2, in fn' + buf.append(String.format("\tFile \"%s\", ", fr.location.file())); + if (fr.location.line() != 0) { + buf.append("line ").append(fr.location.line()).append(", "); + if (fr.location.column() != 0) { + buf.append("column ").append(fr.location.column()).append(", "); + } + } + buf.append("in ").append(fr.name).append('\n'); + + // source line + String line = src.readline(fr.location); + if (line != null) { + buf.append("\t\t").append(line.trim()).append('\n'); + } + } + } + buf.append(prefix).append(message); + return buf.toString(); + } + + // Ensures that this exception holds a call stack, taking the current + // stack (which must be non-empty) from the thread if not. + @CanIgnoreReturnValue + final EvalException ensureStack(StarlarkThread thread) { + if (callstack == null) { + this.callstack = thread.getCallStack(); + if (callstack.isEmpty()) { + throw new IllegalStateException("empty callstack"); + } + } + return this; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/EvalUtils.java b/third_party/bazel/main/java/net/starlark/java/eval/EvalUtils.java new file mode 100644 index 000000000..bc8053da7 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/EvalUtils.java @@ -0,0 +1,531 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.eval; + +import java.util.IllegalFormatException; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nullable; +import net.starlark.java.syntax.TokenKind; + +/** Internal declarations used by the evaluator. */ +final class EvalUtils { + + private EvalUtils() {} + + static void addIterator(Object x) { + if (x instanceof Mutability.Freezable) { + ((Mutability.Freezable) x).updateIteratorCount(+1); + } + } + + static void removeIterator(Object x) { + if (x instanceof Mutability.Freezable) { + ((Mutability.Freezable) x).updateIteratorCount(-1); + } + } + + // The following functions for indexing and slicing match the behavior of Python. + + /** + * Resolves a positive or negative index to an index in the range [0, length), or throws + * EvalException if it is out of range. If the index is negative, it counts backward from length. + */ + static int getSequenceIndex(int index, int length) throws EvalException { + int actualIndex = index; + if (actualIndex < 0) { + actualIndex += length; + } + if (actualIndex < 0 || actualIndex >= length) { + throw Starlark.errorf( + "index out of range (index is %d, but sequence has %d elements)", index, length); + } + return actualIndex; + } + + /** + * Returns the effective index denoted by a user-supplied integer. First, if the integer is + * negative, the length of the sequence is added to it, so an index of -1 represents the last + * element of the sequence. Then, the integer is "clamped" into the inclusive interval [0, + * length]. + */ + static int toIndex(int index, int length) { + if (index < 0) { + index += length; + } + + if (index < 0) { + return 0; + } else if (index > length) { + return length; + } else { + return index; + } + } + + /** Evaluates an eager binary operation, {@code x op y}. (Excludes AND and OR.) */ + static Object binaryOp(TokenKind op, Object x, Object y, StarlarkThread starlarkThread) + throws EvalException { + StarlarkSemantics semantics = starlarkThread.getSemantics(); + Mutability mu = starlarkThread.mutability(); + switch (op) { + case PLUS: + if (x instanceof StarlarkInt) { + if (y instanceof StarlarkInt) { + // int + int + return StarlarkInt.add((StarlarkInt) x, (StarlarkInt) y); + } else if (y instanceof StarlarkFloat) { + // int + float + double z = ((StarlarkInt) x).toFiniteDouble() + ((StarlarkFloat) y).toDouble(); + return StarlarkFloat.of(z); + } + + } else if (x instanceof String) { + if (y instanceof String) { + // string + string + return (String) x + (String) y; + } + + } else if (x instanceof Tuple) { + if (y instanceof Tuple) { + // tuple + tuple + return Tuple.concat((Tuple) x, (Tuple) y); + } + + } else if (x instanceof StarlarkList) { + if (y instanceof StarlarkList) { + // list + list + return StarlarkList.concat((StarlarkList) x, (StarlarkList) y, mu); + } + + } else if (x instanceof StarlarkFloat) { + double xf = ((StarlarkFloat) x).toDouble(); + if (y instanceof StarlarkFloat) { + // float + float + double z = xf + ((StarlarkFloat) y).toDouble(); + return StarlarkFloat.of(z); + } else if (y instanceof StarlarkInt) { + // float + int + double z = xf + ((StarlarkInt) y).toFiniteDouble(); + return StarlarkFloat.of(z); + } + } + break; + + case PIPE: + if (x instanceof StarlarkInt) { + if (y instanceof StarlarkInt) { + // int | int + return StarlarkInt.or((StarlarkInt) x, (StarlarkInt) y); + } + } else if (x instanceof Map) { + if (y instanceof Map) { + // map | map (usually dicts) + return Dict.builder().putAll((Map) x).putAll((Map) y).build(mu); + } + } else if (x instanceof Set && y instanceof Set) { + // set | set + if (semantics.getBool(StarlarkSemantics.EXPERIMENTAL_ENABLE_STARLARK_SET)) { + return StarlarkSet.empty().union(Tuple.of(x, y), starlarkThread); + } + } + break; + + case AMPERSAND: + if (x instanceof StarlarkInt && y instanceof StarlarkInt) { + // int & int + return StarlarkInt.and((StarlarkInt) x, (StarlarkInt) y); + } else if (x instanceof Set && y instanceof Set) { + // set & set + if (semantics.getBool(StarlarkSemantics.EXPERIMENTAL_ENABLE_STARLARK_SET)) { + StarlarkSet xSet = + x instanceof StarlarkSet ? (StarlarkSet) x : StarlarkSet.checkedCopyOf(mu, x); + return xSet.intersection(Tuple.of(y), starlarkThread); + } + } + break; + + case CARET: + if (x instanceof StarlarkInt && y instanceof StarlarkInt) { + // int ^ int + return StarlarkInt.xor((StarlarkInt) x, (StarlarkInt) y); + } else if (x instanceof Set && y instanceof Set) { + // set ^ set + if (semantics.getBool(StarlarkSemantics.EXPERIMENTAL_ENABLE_STARLARK_SET)) { + StarlarkSet xSet = + x instanceof StarlarkSet ? (StarlarkSet) x : StarlarkSet.checkedCopyOf(mu, x); + return xSet.symmetricDifference(y, starlarkThread); + } + } + break; + + case GREATER_GREATER: + if (x instanceof StarlarkInt && y instanceof StarlarkInt) { + // x >> y + return StarlarkInt.shiftRight((StarlarkInt) x, (StarlarkInt) y); + } + break; + + case LESS_LESS: + if (x instanceof StarlarkInt && y instanceof StarlarkInt) { + // x << y + return StarlarkInt.shiftLeft((StarlarkInt) x, (StarlarkInt) y); + } + break; + + case MINUS: + if (x instanceof StarlarkInt) { + if (y instanceof StarlarkInt) { + // int - int + return StarlarkInt.subtract((StarlarkInt) x, (StarlarkInt) y); + } else if (y instanceof StarlarkFloat) { + // int - float + double z = ((StarlarkInt) x).toFiniteDouble() - ((StarlarkFloat) y).toDouble(); + return StarlarkFloat.of(z); + } + + } else if (x instanceof StarlarkFloat) { + double xf = ((StarlarkFloat) x).toDouble(); + if (y instanceof StarlarkFloat) { + // float - float + double z = xf - ((StarlarkFloat) y).toDouble(); + return StarlarkFloat.of(z); + } else if (y instanceof StarlarkInt) { + // float - int + double z = xf - ((StarlarkInt) y).toFiniteDouble(); + return StarlarkFloat.of(z); + } + } else if (x instanceof Set && y instanceof Set) { + // set - set + if (semantics.getBool(StarlarkSemantics.EXPERIMENTAL_ENABLE_STARLARK_SET)) { + StarlarkSet xSet = + x instanceof StarlarkSet ? (StarlarkSet) x : StarlarkSet.checkedCopyOf(mu, x); + return xSet.difference(Tuple.of(y), starlarkThread); + } + } + break; + + case STAR: + if (x instanceof StarlarkInt xi) { + if (y instanceof StarlarkInt) { + // int * int + return StarlarkInt.multiply(xi, (StarlarkInt) y); + } else if (y instanceof String) { + // int * string + return repeatString((String) y, xi); + } else if (y instanceof Tuple) { + // int * tuple + return ((Tuple) y).repeat(xi); + } else if (y instanceof StarlarkList) { + // int * list + return ((StarlarkList) y).repeat(xi, mu); + } else if (y instanceof StarlarkFloat) { + // int * float + double z = xi.toFiniteDouble() * ((StarlarkFloat) y).toDouble(); + return StarlarkFloat.of(z); + } + + } else if (x instanceof String) { + if (y instanceof StarlarkInt) { + // string * int + return repeatString((String) x, (StarlarkInt) y); + } + + } else if (x instanceof Tuple) { + if (y instanceof StarlarkInt) { + // tuple * int + return ((Tuple) x).repeat((StarlarkInt) y); + } + + } else if (x instanceof StarlarkList) { + if (y instanceof StarlarkInt) { + // list * int + return ((StarlarkList) x).repeat((StarlarkInt) y, mu); + } + + } else if (x instanceof StarlarkFloat) { + double xf = ((StarlarkFloat) x).toDouble(); + if (y instanceof StarlarkFloat) { + // float * float + return StarlarkFloat.of(xf * ((StarlarkFloat) y).toDouble()); + } else if (y instanceof StarlarkInt) { + // float * int + return StarlarkFloat.of(xf * ((StarlarkInt) y).toFiniteDouble()); + } + } + break; + + case SLASH: // real division + if (x instanceof StarlarkInt) { + double xf = ((StarlarkInt) x).toFiniteDouble(); + if (y instanceof StarlarkInt) { + // int / int + return StarlarkFloat.div(xf, ((StarlarkInt) y).toFiniteDouble()); + } else if (y instanceof StarlarkFloat) { + // int / float + return StarlarkFloat.div(xf, ((StarlarkFloat) y).toDouble()); + } + + } else if (x instanceof StarlarkFloat) { + double xf = ((StarlarkFloat) x).toDouble(); + if (y instanceof StarlarkFloat) { + // float / float + return StarlarkFloat.div(xf, ((StarlarkFloat) y).toDouble()); + } else if (y instanceof StarlarkInt) { + // float / int + return StarlarkFloat.div(xf, ((StarlarkInt) y).toFiniteDouble()); + } + } + break; + + case SLASH_SLASH: + if (x instanceof StarlarkInt) { + if (y instanceof StarlarkInt) { + // int // int + return StarlarkInt.floordiv((StarlarkInt) x, (StarlarkInt) y); + } else if (y instanceof StarlarkFloat) { + // int // float + double xf = ((StarlarkInt) x).toFiniteDouble(); + double yf = ((StarlarkFloat) y).toDouble(); + return StarlarkFloat.floordiv(xf, yf); + } + + } else if (x instanceof StarlarkFloat) { + double xf = ((StarlarkFloat) x).toDouble(); + if (y instanceof StarlarkFloat) { + // float // float + return StarlarkFloat.floordiv(xf, ((StarlarkFloat) y).toDouble()); + } else if (y instanceof StarlarkInt) { + // float // int + return StarlarkFloat.floordiv(xf, ((StarlarkInt) y).toFiniteDouble()); + } + } + break; + + case PERCENT: + if (x instanceof StarlarkInt) { + if (y instanceof StarlarkInt) { + // int % int + return StarlarkInt.mod((StarlarkInt) x, (StarlarkInt) y); + + } else if (y instanceof StarlarkFloat) { + // int % float + double xf = ((StarlarkInt) x).toFiniteDouble(); + double yf = ((StarlarkFloat) y).toDouble(); + return StarlarkFloat.mod(xf, yf); + } + + } else if (x instanceof String xs) { + // string % any + try { + if (y instanceof Tuple) { + return Starlark.formatWithList(semantics, xs, (Tuple) y); + } else { + return Starlark.format(semantics, xs, y); + } + } catch (IllegalFormatException ex) { + throw new EvalException(ex); + } + + } else if (x instanceof StarlarkFloat) { + double xf = ((StarlarkFloat) x).toDouble(); + if (y instanceof StarlarkFloat) { + // float % float + return StarlarkFloat.mod(xf, ((StarlarkFloat) y).toDouble()); + } else if (y instanceof StarlarkInt) { + // float % int + return StarlarkFloat.mod(xf, ((StarlarkInt) y).toFiniteDouble()); + } + } + break; + + case EQUALS_EQUALS: + return x.equals(y); + + case NOT_EQUALS: + return !x.equals(y); + + case LESS: + return compare(x, y) < 0; + + case LESS_EQUALS: + return compare(x, y) <= 0; + + case GREATER: + return compare(x, y) > 0; + + case GREATER_EQUALS: + return compare(x, y) >= 0; + + case IN: + if (y instanceof StarlarkMembershipTestable) { + return ((StarlarkMembershipTestable) y).containsKey(semantics, x); + } else if (y instanceof StarlarkIndexable.Threaded) { + return ((StarlarkIndexable.Threaded) y).containsKey(starlarkThread, semantics, x); + } else if (y instanceof String) { + if (!(x instanceof String)) { + throw Starlark.errorf( + "'in ' requires string as left operand, not '%s'", Starlark.type(x)); + } + return ((String) y).contains((String) x); + } + break; + + case NOT_IN: + Object z = binaryOp(TokenKind.IN, x, y, starlarkThread); + if (z != null) { + return !Starlark.truth(z); + } + break; + + default: + throw new AssertionError("not a binary operator: " + op); + } + + // custom binary operator? + if (x instanceof HasBinary) { + Object z = ((HasBinary) x).binaryOp(op, y, true); + if (z != null) { + return z; + } + } + if (y instanceof HasBinary) { + Object z = ((HasBinary) y).binaryOp(op, x, false); + if (z != null) { + return z; + } + } + + throw Starlark.errorf( + "unsupported binary operation: %s %s %s", Starlark.type(x), op, Starlark.type(y)); + } + + // Defines the behavior of the language's ordered comparison operators (< <= => >). + private static int compare(Object x, Object y) throws EvalException { + try { + return Starlark.compareUnchecked(x, y); + } catch (ClassCastException ex) { + throw new EvalException(ex.getMessage()); + } + } + + private static String repeatString(String s, StarlarkInt in) throws EvalException { + int n = in.toInt("repeat"); + if (n <= 0) { + return ""; + } else if ((long) s.length() * (long) n > Integer.MAX_VALUE) { + // Would exceed max length of a java String. + throw Starlark.errorf("excessive repeat (%d * %d characters)", s.length(), n); + } else { + return s.repeat(n); + } + } + + /** Evaluates a unary operation. */ + static Object unaryOp(TokenKind op, Object x) throws EvalException { + switch (op) { + case NOT: + return !Starlark.truth(x); + + case MINUS: + if (x instanceof StarlarkInt) { + return StarlarkInt.uminus((StarlarkInt) x); // -int + } else if (x instanceof StarlarkFloat) { + return StarlarkFloat.of(-((StarlarkFloat) x).toDouble()); // -float + } + break; + + case PLUS: + if (x instanceof StarlarkInt) { + return x; // +int + } else if (x instanceof StarlarkFloat) { + return x; // +float + } + break; + + case TILDE: + if (x instanceof StarlarkInt) { + return StarlarkInt.bitnot((StarlarkInt) x); // ~int + } + break; + + default: + /* fall through */ + } + throw Starlark.errorf("unsupported unary operation: %s%s", op, Starlark.type(x)); + } + + /** + * Returns the element of sequence or mapping {@code object} indexed by {@code key}. + * + * @throws EvalException if {@code object} is not a sequence or mapping. + */ + @Nullable + static Object index(StarlarkThread starlarkThread, Object object, Object key) + throws EvalException { + Mutability mu = starlarkThread.mutability(); + StarlarkSemantics semantics = starlarkThread.getSemantics(); + + if (object instanceof StarlarkIndexable.Threaded) { + return ((StarlarkIndexable.Threaded) object).getIndex(starlarkThread, semantics, key); + } else if (object instanceof StarlarkIndexable) { + Object result = ((StarlarkIndexable) object).getIndex(semantics, key); + // TODO(bazel-team): We shouldn't have this fromJava call here. If it's needed at all, + // it should go in the implementations of StarlarkIndexable#getIndex that produce non-Starlark + // values. + return result == null ? null : Starlark.fromJava(result, mu); + } else if (object instanceof String string) { + int index = Starlark.toInt(key, "string index"); + index = getSequenceIndex(index, string.length()); + return StringModule.memoizedCharToString(string.charAt(index)); + } else { + throw Starlark.errorf( + "type '%s' has no operator [](%s)", Starlark.type(object), Starlark.type(key)); + } + } + + /** + * Updates an object as if by the Starlark statement {@code object[key] = value}. + * + * @throws EvalException if the object is not a list or dict. + */ + static void setIndex(Object object, Object key, Object value) throws EvalException { + if (object instanceof Dict) { + @SuppressWarnings("unchecked") + Dict dict = (Dict) object; + dict.putEntry(key, value); + + } else if (object instanceof StarlarkList) { + @SuppressWarnings("unchecked") + StarlarkList list = (StarlarkList) object; + int index = Starlark.toInt(key, "list index"); + index = EvalUtils.getSequenceIndex(index, list.size()); + list.setElementAt(index, value); + + } else { + throw Starlark.errorf( + "can only assign an element in a dictionary or a list, not in a '%s'", + Starlark.type(object)); + } + } + + /** Updates the named field of x as if by the Starlark statement {@code x.field = value}. */ + static void setField(Object x, String field, Object value) throws EvalException { + if (x instanceof Structure) { + ((Structure) x).setField(field, value); + } else { + throw Starlark.errorf("cannot set .%s field of %s value", field, Starlark.type(x)); + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/FlagGuardedValue.java b/third_party/bazel/main/java/net/starlark/java/eval/FlagGuardedValue.java new file mode 100644 index 000000000..8936620f0 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/FlagGuardedValue.java @@ -0,0 +1,87 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +/** {@link GuardedValue} that controls access based on an experimental or incompatible flag. */ +public final class FlagGuardedValue { + /** + * Creates a flag guard which only permits access of the given object when the given boolean flag + * is true. If the given flag is false and the object would be accessed, an error is thrown + * describing the feature as experimental, and describing that the flag must be set to true. + * + *

The flag identifier must have a + or - prefix; see StarlarkSemantics. + */ + public static GuardedValue onlyWhenExperimentalFlagIsTrue(String flag, Object obj) { + if (flag.charAt(0) != '-' && flag.charAt(0) != '+') { + throw new IllegalArgumentException(String.format("flag needs [+-] prefix: %s", flag)); + } + return new GuardedValue() { + @Override + public Object getObject() { + return obj; + } + + @Override + public String getErrorFromAttemptingAccess(String name) { + return name + + " is experimental and thus unavailable with the current flags. It may be enabled by" + + " setting --" + + flag.substring(1); + } + + @Override + public boolean isObjectAccessibleUsingSemantics( + StarlarkSemantics semantics, Object clientData) { + return semantics.isFeatureEnabledBasedOnTogglingFlags(flag, ""); + } + }; + } + + /** + * Creates a flag guard which only permits access of the given object when the given boolean flag + * is false. If the given flag is true and the object would be accessed, an error is thrown + * describing the feature as deprecated, and describing that the flag must be set to false. + * + *

The flag identifier must have a + or - prefix; see StarlarkSemantics. + */ + public static GuardedValue onlyWhenIncompatibleFlagIsFalse(String flag, Object obj) { + if (flag.charAt(0) != '-' && flag.charAt(0) != '+') { + throw new IllegalArgumentException(String.format("flag needs [+-] prefix: %s", flag)); + } + return new GuardedValue() { + @Override + public Object getObject() { + return obj; + } + + @Override + public String getErrorFromAttemptingAccess(String name) { + return name + + " is deprecated and will be removed soon. It may be temporarily re-enabled by" + + " setting --" + + flag.substring(1) + + "=false"; + } + + @Override + public boolean isObjectAccessibleUsingSemantics( + StarlarkSemantics semantics, Object clientData) { + return semantics.isFeatureEnabledBasedOnTogglingFlags("", flag); + } + }; + } + + private FlagGuardedValue() {} +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/FormatParser.java b/third_party/bazel/main/java/net/starlark/java/eval/FormatParser.java new file mode 100644 index 000000000..c0e717b59 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/FormatParser.java @@ -0,0 +1,294 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.eval; + +import com.google.common.base.CharMatcher; +import com.google.common.collect.ImmutableSet; +import java.util.List; +import java.util.Map; + +/** + * A helper class that offers a subset of the functionality of Python's string#format. + * + *

Currently, both manual and automatic positional as well as named replacement fields are + * supported. However, nested replacement fields are not allowed. + */ +final class FormatParser { + + /** + * Matches strings likely to be a number, faster alternative to relying solely on Integer.parseInt + * and NumberFormatException to determine numericness. + */ + private static final CharMatcher LIKELY_NUMERIC_MATCHER = + CharMatcher.inRange('0', '9').or(CharMatcher.is('-')); + + private static final ImmutableSet ILLEGAL_IN_FIELD = + ImmutableSet.of('.', '[', ']', ','); + + /** + * Formats the given input string by using the given arguments + * + *

This method offers a subset of the functionality of Python's string#format + * + * @param input The string to be formatted + * @param args Positional arguments + * @param kwargs Named arguments + * @return The formatted string + */ + String format( + String input, List args, Map kwargs, StarlarkSemantics semantics) + throws EvalException { + char[] chars = input.toCharArray(); + StringBuilder output = new StringBuilder(); + History history = new History(); + + for (int pos = 0; pos < chars.length; ++pos) { + char current = chars[pos]; + int advancePos = 0; + + if (current == '{') { + advancePos = processOpeningBrace(chars, pos, args, kwargs, history, output, semantics); + } else if (current == '}') { + advancePos = processClosingBrace(chars, pos, output); + } else { + output.append(current); + } + + pos += advancePos; + } + + return output.toString(); + } + + /** + * Processes the expression after an opening brace (possibly a replacement field) and emits the + * result to the output StringBuilder + * + * @param chars The entire string + * @param pos The position of the opening brace + * @param args List of positional arguments + * @param kwargs Map of named arguments + * @param history Helper object that tracks information about previously seen positional + * replacement fields + * @param output StringBuilder that consumes the result + * @return Number of characters that have been consumed by this method + */ + private int processOpeningBrace( + char[] chars, + int pos, + List args, + Map kwargs, + History history, + StringBuilder output, + StarlarkSemantics semantics) + throws EvalException { + Printer printer = new Printer(output); + if (has(chars, pos + 1, '{')) { + // Escaped brace -> output and move to char after right brace + printer.append("{"); + return 1; + } + + // Inside a replacement field + String key = getFieldName(chars, pos); + Object value = null; + + // Only positional replacement fields will lead to a valid index + try { + if (key.isEmpty() || LIKELY_NUMERIC_MATCHER.matchesAllOf(key)) { + int index = parsePositional(key, history); + + if (index < 0 || index >= args.size()) { + throw Starlark.errorf("No replacement found for index %d", index); + } + + value = args.get(index); + } else { + value = getKwarg(kwargs, key); + } + } catch (NumberFormatException nfe) { + // Non-integer index -> Named + value = getKwarg(kwargs, key); + } + + // Format object for output + printer.str(value, semantics); + + // Advances the current position to the index of the closing brace of the + // replacement field. Due to the definition of the enclosing for() loop, + // the next iteration will examine the character right after the brace. + return key.length() + 1; + } + + private Object getKwarg(Map kwargs, String key) throws EvalException { + if (!kwargs.containsKey(key)) { + throw Starlark.errorf("Missing argument '%s'", key); + } + + return kwargs.get(key); + } + + /** + * Processes a closing brace and emits the result to the output StringBuilder + * + * @param chars The entire string + * @param pos Position of the closing brace + * @param output StringBuilder that consumes the result + * @return Number of characters that have been consumed by this method + */ + private int processClosingBrace(char[] chars, int pos, StringBuilder output) + throws EvalException { + if (!has(chars, pos + 1, '}')) { + // Invalid brace outside replacement field + throw Starlark.errorf("Found '}' without matching '{'"); + } + + // Escaped brace -> output and move to char after right brace + output.append("}"); + return 1; + } + + /** + * Checks whether the given input string has a specific character at the given location + * + * @param data Input string as character array + * @param pos Position to be checked + * @param needle Character to be searched for + * @return True if string has the specified character at the given location + */ + private static boolean has(char[] data, int pos, char needle) { + return pos < data.length && data[pos] == needle; + } + + /** + * Extracts the name/index of the replacement field that starts at the specified location + * + * @param chars Input string + * @param openingBrace Position of the opening brace of the replacement field + * @return Name or index of the current replacement field + */ + private String getFieldName(char[] chars, int openingBrace) throws EvalException { + StringBuilder result = new StringBuilder(); + boolean foundClosingBrace = false; + + for (int pos = openingBrace + 1; pos < chars.length; ++pos) { + char current = chars[pos]; + + if (current == '}') { + foundClosingBrace = true; + break; + } else { + if (current == '{') { + throw Starlark.errorf("Nested replacement fields are not supported"); + } else if (ILLEGAL_IN_FIELD.contains(current)) { + throw Starlark.errorf("Invalid character '%s' inside replacement field", current); + } + + result.append(current); + } + } + + if (!foundClosingBrace) { + throw Starlark.errorf("Found '{' without matching '}'"); + } + + return result.toString(); + } + + /** + * Converts the given key into an integer or assigns the next available index, if empty. + * + * @param key Key to be converted + * @param history Helper object that tracks information about previously seen positional + * replacement fields + * @return The integer equivalent of the key + */ + private int parsePositional(String key, History history) throws EvalException { + int result = -1; + + try { + if (key.isEmpty()) { + // Automatic positional -> a new index value has to be assigned + history.setAutomaticPositional(); + result = history.getNextPosition(); + } else { + // This will fail if key is a named argument + result = Integer.parseInt(key); + history.setManualPositional(); // Only register if the conversion succeeds + } + } catch (MixedTypeException mte) { + throw Starlark.errorf("%s", mte.getMessage()); + } + + return result; + } + + /** + * Exception for invalid combinations of replacement field types + */ + private static final class MixedTypeException extends Exception { + MixedTypeException() { + super("Cannot mix manual and automatic numbering of positional fields"); + } + } + + /** + * A wrapper to keep track of information about previous replacement fields + */ + private static final class History { + /** Different types of positional replacement fields */ + enum Positional { + NONE, + MANUAL, // {0}, {1} etc. + AUTOMATIC // {} + } + + Positional type = Positional.NONE; + int position = -1; + + /** + * Returns the next available index for an automatic positional replacement field + * + * @return Next index + */ + int getNextPosition() { + ++position; + return position; + } + + /** Registers a manual positional replacement field */ + void setManualPositional() throws MixedTypeException { + setPositional(Positional.MANUAL); + } + + /** Registers an automatic positional replacement field */ + void setAutomaticPositional() throws MixedTypeException { + setPositional(Positional.AUTOMATIC); + } + + /** + * Indicates that a positional replacement field of the specified type is being processed and + * checks whether this conflicts with any previously seen replacement fields + * + * @param current Type of current replacement field + */ + void setPositional(Positional current) throws MixedTypeException { + if (type == Positional.NONE) { + type = current; + } else if (type != current) { + throw new MixedTypeException(); + } + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/GuardedValue.java b/third_party/bazel/main/java/net/starlark/java/eval/GuardedValue.java new file mode 100644 index 000000000..cb0251f50 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/GuardedValue.java @@ -0,0 +1,49 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import javax.annotation.Nullable; + +/** + * Wrapper interface on a value in the predeclared lexical block, that controls its accessibility to + * Starlark based on the value of a semantic flag and/or the Module's client data. + * + *

For example, this could control whether symbol "Foo" exists in the Starlark global frame: such + * a symbol might only be accessible if --experimental_foo is set to true. In order to create this + * control, an instance of this class should be added to the global frame under name "Foo". This + * guard will throw a descriptive {@link EvalException} when "Foo" would be accessed without the + * proper flag. + */ +public interface GuardedValue { + + /** + * Returns an error describing an attempt to access this guard's protected object when it should + * be inaccessible under the (contextually implied) semantics and client data. + */ + String getErrorFromAttemptingAccess(String name); + + /** + * Returns this guard's underlying object. This should be called when appropriate validation has + * occurred to ensure that the object is accessible with the (implied) semantics. + */ + Object getObject(); + + /** + * Returns true if this guard's underlying object is accessible under the given semantics and + * client data. + */ + boolean isObjectAccessibleUsingSemantics( + StarlarkSemantics semantics, @Nullable Object clientData); +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/HasBinary.java b/third_party/bazel/main/java/net/starlark/java/eval/HasBinary.java new file mode 100644 index 000000000..d55702f44 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/HasBinary.java @@ -0,0 +1,41 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import javax.annotation.Nullable; +import net.starlark.java.syntax.TokenKind; + +/** + * A Starlark value that supports binary operators such as {@code x+y}. + * + *

During evaluation of a Starlark binary operation, if none of the built-in cases match, then + * the left operand is queried; if it implements HasBinary, its {@link #binaryOp} method is called. + * If the left operand does not implement HasBinary, or declines to implement the particular + * operation by returning null, then the right operand is queried for HasBinary and its {@link + * #binaryOp} method is called. If neither operand defines the operator, evaluation fails. + * + *

Subclasses should strive for appropriate symmetries in their implementations, such as {@code x + * * y == y * x}. + */ +// TODO(adonovan): rename BinaryOperand? +public interface HasBinary extends StarlarkValue { + + /** + * Returns {@code this op that}, if thisLeft, or {@code that op this} otherwise. May return null + * to indicate that the operation is not supported, or may throw a specific exception. + */ + @Nullable + Object binaryOp(TokenKind op, Object that, boolean thisLeft) throws EvalException; +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/ImmutableSingletonStarlarkList.java b/third_party/bazel/main/java/net/starlark/java/eval/ImmutableSingletonStarlarkList.java new file mode 100644 index 000000000..29fa355f7 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/ImmutableSingletonStarlarkList.java @@ -0,0 +1,78 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import java.util.Arrays; + +/** An immutable singleton implementation of a {@code StarlarkList}. */ +final class ImmutableSingletonStarlarkList extends ImmutableStarlarkList { + private final Object elem; + + ImmutableSingletonStarlarkList(Object elem) { + this.elem = elem; + } + + @Override + @SuppressWarnings("unchecked") + public ImmutableList getImmutableList() { + return ImmutableList.of((E) elem); + } + + @Override + @SuppressWarnings("unchecked") + public E get(int i) { + Preconditions.checkElementIndex(i, 1); + return (E) elem; // unchecked + } + + @Override + public int size() { + return 1; + } + + @Override + public boolean contains(Object o) { + // StarlarkList contains only valid Starlark objects (which are non-null) + if (o == null) { + return false; + } + return o.equals(elem); + } + + /** Returns a new array of class Object[] containing the list elements. */ + @Override + public Object[] toArray() { + return new Object[] {elem}; + } + + @SuppressWarnings("unchecked") + @Override + public T[] toArray(T[] a) { + if (a.length < 1) { + return (T[]) new Object[] {elem}; + } else { + a[0] = (T) elem; + Arrays.fill(a, 1, a.length, null); + return a; + } + } + + @Override + Object[] elems() { + return new Object[] {elem}; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/ImmutableStarlarkList.java b/third_party/bazel/main/java/net/starlark/java/eval/ImmutableStarlarkList.java new file mode 100644 index 000000000..3a8306ca2 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/ImmutableStarlarkList.java @@ -0,0 +1,130 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import java.util.Arrays; + +/** Partial implementation of an immutable {@code StarlarkList}. */ +abstract class ImmutableStarlarkList extends StarlarkList { + + @Override + public final boolean isImmutable() { + return true; + } + + @Override + public final boolean updateIteratorCount(int delta) { + return false; + } + + @Override + public final Mutability mutability() { + return Mutability.IMMUTABLE; + } + + @Override + public final void unsafeShallowFreeze() { + Mutability.Freezable.checkUnsafeShallowFreezePrecondition(this); + } + + @Override + public final void addElement(E element) throws EvalException { + Starlark.checkMutable(this); + } + + @Override + public final void addElementAt(int index, E element) throws EvalException { + Starlark.checkMutable(this); + } + + @Override + public final void addElements(Iterable elements) throws EvalException { + Starlark.checkMutable(this); + } + + @Override + public final void removeElementAt(int index) throws EvalException { + Starlark.checkMutable(this); + } + + @Override + public final void setElementAt(int index, E value) throws EvalException { + Starlark.checkMutable(this); + } + + @Override + public final void clearElements() throws EvalException { + Starlark.checkMutable(this); + } + + @Override + public ImmutableList getImmutableList() { + // Optimization: a frozen array needn't be copied. + // If the entire array is full, we can wrap it directly. + return Tuple.wrapImmutable(elems()); + } + + @Override + @SuppressWarnings("unchecked") + public E get(int i) { + Object[] elems = elems(); + Preconditions.checkElementIndex(i, elems.length); + return (E) elems[i]; // unchecked + } + + @Override + public int size() { + return elems().length; + } + + @Override + public boolean contains(Object o) { + // StarlarkList contains only valid Starlark objects (which are non-null) + if (o == null) { + return false; + } + Object[] elems = elems(); + int size = elems.length; + for (int i = 0; i < size; i++) { + Object elem = elems[i]; + if (o.equals(elem)) { + return true; + } + } + return false; + } + + /** Returns a new array of class Object[] containing the list elements. */ + @Override + public Object[] toArray() { + Object[] elems = elems(); + return elems.length != 0 ? Arrays.copyOf(elems, elems.length, Object[].class) : EMPTY_ARRAY; + } + + @SuppressWarnings("unchecked") + @Override + public T[] toArray(T[] a) { + Object[] elems = elems(); + if (a.length < elems.length) { + return (T[]) Arrays.copyOf(elems, elems.length, a.getClass()); + } else { + System.arraycopy(elems, 0, a, 0, elems.length); + Arrays.fill(a, elems.length, a.length, null); + return a; + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/JNI.java b/third_party/bazel/main/java/net/starlark/java/eval/JNI.java new file mode 100644 index 000000000..6c2d1ce73 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/JNI.java @@ -0,0 +1,30 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +final class JNI { + private JNI() {} // uninstantiable + + static void load() { + try { + System.loadLibrary("cpu_profiler"); + } catch (UnsatisfiedLinkError ex) { + // Ignore, deferring the error until a C function is called, if ever. + // Without this hack //src/test/shell/bazel:bazel_bootstrap_distfile_test + // fails with an utterly uninformative error. + // TODO(adonovan): remove try/catch once that test is fixed. + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/LazyImmutableStarlarkList.java b/third_party/bazel/main/java/net/starlark/java/eval/LazyImmutableStarlarkList.java new file mode 100644 index 000000000..4ba88f402 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/LazyImmutableStarlarkList.java @@ -0,0 +1,61 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.base.Preconditions; + +/** An immutable {@link StarlarkList} that lazily invokes a supplier to obtain its elements. */ +public final class LazyImmutableStarlarkList extends ImmutableStarlarkList { + private SerializableListSupplier supplier; + private volatile Object[] elems; + + LazyImmutableStarlarkList(SerializableListSupplier supplier) { + this.supplier = supplier; + } + + @Override + public int size() { + return elems().length; + } + + @Override + @SuppressWarnings("unchecked") + public E get(int i) { + Object[] elems = elems(); + Preconditions.checkElementIndex(i, elems.length); + return (E) elems[i]; + } + + @Override + Object[] elems() { + if (elems == null) { + synchronized (this) { + if (elems == null) { + elems = supplier.get().toArray(); + supplier = null; + } + } + } + return elems; + } + + @Override + public StarlarkList unsafeOptimizeMemoryLayout() { + if (elems != null) { + return StarlarkList.wrap(Mutability.IMMUTABLE, elems); + } + return this; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/MethodDescriptor.java b/third_party/bazel/main/java/net/starlark/java/eval/MethodDescriptor.java new file mode 100644 index 000000000..627c3a846 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/MethodDescriptor.java @@ -0,0 +1,320 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import static java.util.Arrays.stream; + +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.errorprone.annotations.CheckReturnValue; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; +import javax.annotation.Nullable; +import net.starlark.java.annot.Param; +import net.starlark.java.annot.StarlarkMethod; + +/** + * A value class to store Methods with their corresponding {@link StarlarkMethod} annotation + * metadata. This is needed because the annotation is sometimes in a superclass. + * + *

The annotation metadata is duplicated in this class to avoid usage of Java dynamic proxies + * which are ~7× slower. + */ +final class MethodDescriptor { + private final Method method; + private final StarlarkMethod annotation; + + private final String name; + private final String doc; + private final boolean documented; + private final boolean structField; + private final ParamDescriptor[] parameters; + private final boolean extraPositionals; + private final boolean extraKeywords; + private final boolean selfCall; + private final boolean allowReturnNones; + private final boolean useStarlarkThread; + private final boolean useStarlarkSemantics; + private final boolean positionalsReusableAsJavaArgsVectorIfArgumentCountValid; + + private enum HowToHandleReturn { + NULL_TO_NONE, // any Starlark value; null -> None + ERROR_ON_NULL, // any Starlark value; null -> error + STARLARK_INT_OF_INT, // Java int -> StarlarkInt + FROM_JAVA, // Starlark.fromJava conversion (List, Map, various Numbers, null perhaps) + } + + private final HowToHandleReturn howToHandleReturn; + + private MethodDescriptor( + Method method, + StarlarkMethod annotation, + String name, + String doc, + boolean documented, + boolean structField, + ParamDescriptor[] parameters, + boolean extraPositionals, + boolean extraKeywords, + boolean selfCall, + boolean allowReturnNones, + boolean useStarlarkThread, + boolean useStarlarkSemantics) { + this.method = method; + this.annotation = annotation; + this.name = name; + this.doc = doc; + this.documented = documented; + this.structField = structField; + this.parameters = parameters; + this.extraPositionals = extraPositionals; + this.extraKeywords = extraKeywords; + this.selfCall = selfCall; + this.allowReturnNones = allowReturnNones; + this.useStarlarkThread = useStarlarkThread; + this.useStarlarkSemantics = useStarlarkSemantics; + + Class ret = method.getReturnType(); + if (ret == void.class || ret == boolean.class) { + // * `void` function returns `null` + // * `boolean` function never returns `null` + // We could have specialized enum variant, but null check is cheap. + howToHandleReturn = HowToHandleReturn.NULL_TO_NONE; + } else if (StarlarkValue.class.isAssignableFrom(ret) + || String.class == ret + || Boolean.class == ret) { + howToHandleReturn = + allowReturnNones ? HowToHandleReturn.NULL_TO_NONE : HowToHandleReturn.ERROR_ON_NULL; + } else if (ret == int.class) { + howToHandleReturn = HowToHandleReturn.STARLARK_INT_OF_INT; + } else { + howToHandleReturn = HowToHandleReturn.FROM_JAVA; + } + + this.positionalsReusableAsJavaArgsVectorIfArgumentCountValid = + !extraKeywords + && !extraPositionals + && !useStarlarkSemantics + && !useStarlarkThread + && stream(parameters).allMatch(MethodDescriptor::paramUsableAsPositionalWithoutChecks); + } + + private static boolean paramUsableAsPositionalWithoutChecks(ParamDescriptor param) { + return param.isPositional() + && param.disabledByFlag() == null + && param.getAllowedClasses() == null; + } + + /** Returns the StarlarkMethod annotation corresponding to this method. */ + StarlarkMethod getAnnotation() { + return annotation; + } + + /** @return Starlark method descriptor for provided Java method and signature annotation. */ + static MethodDescriptor of( + Method method, StarlarkMethod annotation, StarlarkSemantics semantics) { + // This happens when the interface is public but the implementation classes + // have reduced visibility. + method.setAccessible(true); + + Class[] paramClasses = method.getParameterTypes(); + Param[] paramAnnots = annotation.parameters(); + ParamDescriptor[] params = new ParamDescriptor[paramAnnots.length]; + Arrays.setAll(params, i -> ParamDescriptor.of(paramAnnots[i], paramClasses[i], semantics)); + + return new MethodDescriptor( + method, + annotation, + annotation.name(), + annotation.doc(), + annotation.documented(), + annotation.structField(), + params, + !annotation.extraPositionals().name().isEmpty(), + !annotation.extraKeywords().name().isEmpty(), + annotation.selfCall(), + annotation.allowReturnNones(), + annotation.useStarlarkThread(), + annotation.useStarlarkSemantics()); + } + + private static final Object[] EMPTY = {}; + + /** Calls this method, which must have {@code structField=true}. */ + Object callField(Object obj, StarlarkSemantics semantics, @Nullable Mutability mu) + throws EvalException, InterruptedException { + if (!structField) { + throw new IllegalStateException("not a struct field: " + name); + } + Object[] args = useStarlarkSemantics ? new Object[] {semantics} : EMPTY; + return call(obj, args, mu); + } + + /** + * Invokes this method using {@code obj} as a target and {@code args} as Java arguments. + * + *

Methods with {@code void} return type return {@code None} following Python convention. + * + *

The Mutability is used if it is necessary to allocate a Starlark copy of a Java result. + */ + Object call(Object obj, Object[] args, @Nullable Mutability mu) + throws EvalException, InterruptedException { + Preconditions.checkNotNull(obj); + Object result; + try { + result = method.invoke(obj, args); + } catch (IllegalAccessException ex) { + // "Can't happen": the annotated processor ensures that annotated methods are accessible. + throw new IllegalStateException(ex); + + } catch (IllegalArgumentException ex) { + // "Can't happen": unexpected type mismatch. + // Show details to aid debugging (see e.g. b/162444744). + StringBuilder buf = new StringBuilder(); + buf.append( + String.format( + "IllegalArgumentException (%s) in Starlark call of %s, obj=%s (%s), args=[", + ex.getMessage(), method, Starlark.repr(obj), Starlark.type(obj))); + String sep = ""; + for (Object arg : args) { + buf.append(String.format("%s%s (%s)", sep, Starlark.repr(arg), Starlark.type(arg))); + sep = ", "; + } + buf.append(']'); + throw new IllegalArgumentException(buf.toString()); + + } catch (InvocationTargetException ex) { + Throwable e = ex.getCause(); + if (e == null) { + throw new IllegalStateException(e); + } + // Don't intercept unchecked exceptions. + Throwables.throwIfUnchecked(e); + if (e instanceof EvalException) { + throw (EvalException) e; + } else if (e instanceof InterruptedException) { + throw (InterruptedException) e; + } else { + // All other checked exceptions (e.g. LabelSyntaxException) are reported to Starlark. + throw new EvalException(e); + } + } + + // This switch is an optimization to reduce the overhead + // of an unconditional null check and fromJava call. + switch (howToHandleReturn) { + case NULL_TO_NONE: + return result != null ? result : Starlark.NONE; + case ERROR_ON_NULL: + if (result == null) { + throw methodInvocationReturnedNull(args); + } + return result; + case STARLARK_INT_OF_INT: + return StarlarkInt.of((Integer) result); + case FROM_JAVA: + if (result == null && !allowReturnNones) { + throw methodInvocationReturnedNull(args); + } + return Starlark.fromJava(result, mu); + } + throw new IllegalStateException("unreachable: " + howToHandleReturn); + } + + @CheckReturnValue // don't forget to throw it + private NullPointerException methodInvocationReturnedNull(Object[] args) { + return new NullPointerException( + "method invocation returned null: " + getName() + Tuple.of(args)); + } + + /** @see StarlarkMethod#name() */ + String getName() { + return name; + } + + Method getMethod() { + return method; + } + + /** @see StarlarkMethod#structField() */ + boolean isStructField() { + return structField; + } + + /** @see StarlarkMethod#useStarlarkThread() */ + boolean isUseStarlarkThread() { + return useStarlarkThread; + } + + /** @see StarlarkMethod#useStarlarkSemantics() */ + boolean isUseStarlarkSemantics() { + return useStarlarkSemantics; + } + + /** @return {@code true} if this method accepts extra arguments ({@code *args}) */ + boolean acceptsExtraArgs() { + return extraPositionals; + } + + /** @see StarlarkMethod#extraKeywords() */ + boolean acceptsExtraKwargs() { + return extraKeywords; + } + + /** @see StarlarkMethod#parameters() */ + ParamDescriptor[] getParameters() { + return parameters; + } + + /** Returns the index of the named parameter or -1 if not found. */ + int getParameterIndex(String name) { + for (int i = 0; i < parameters.length; i++) { + if (parameters[i].getName().equals(name)) { + return i; + } + } + return -1; + } + + /** @see StarlarkMethod#documented() */ + boolean isDocumented() { + return documented; + } + + /** @see StarlarkMethod#doc() */ + String getDoc() { + return doc; + } + + /** @see StarlarkMethod#selfCall() */ + boolean isSelfCall() { + return selfCall; + } + + /** + * Returns true if we may directly reuse the Starlark positionals vector as the Java {@code args} + * vector passed to {@link #call} as long as the Starlark call was made with a valid number of + * arguments. + * + *

More precisely, this means that we do not need to insert extra values into the args vector + * (such as ones corresponding to {@code *args}, {@code **kwargs}, or {@code self} in Starlark), + * and all Starlark parameters are simple positional parameters which cannot be disabled by a flag + * and do not require type checking. + */ + boolean isPositionalsReusableAsJavaArgsVectorIfArgumentCountValid() { + return positionalsReusableAsJavaArgsVectorIfArgumentCountValid; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/MethodLibrary.java b/third_party/bazel/main/java/net/starlark/java/eval/MethodLibrary.java new file mode 100644 index 000000000..512af283f --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/MethodLibrary.java @@ -0,0 +1,984 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import static com.google.common.collect.Streams.stream; +import static java.util.Comparator.comparing; + +import com.google.common.base.Ascii; +import com.google.common.base.Throwables; +import com.google.common.collect.Ordering; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Optional; +import net.starlark.java.annot.Param; +import net.starlark.java.annot.ParamType; +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; + +/** The universal predeclared functions of core Starlark. */ +class MethodLibrary { + + @StarlarkMethod( + name = "min", + doc = + "Returns the smallest one of all given arguments. If only one positional argument is" + + " provided, it must be a non-empty iterable. It is an error if elements are not" + + " comparable (for example int with string), or if no arguments are given." + + "

\n" //
+              + "min(2, 5, 4) == 2\n"
+              + "min([5, 6, 3]) == 3\n"
+              + "min(\"six\", \"three\", \"four\", key = len) == \"six\"  # the shortest\n"
+              + "min([2, -2, -1, 1], key = abs) == -1  # the first encountered with minimal key"
+              + " value\n"
+              + "
", + extraPositionals = @Param(name = "args", doc = "The elements to be checked."), + parameters = { + @Param( + name = "key", + named = true, + positional = false, + allowedTypes = { + @ParamType(type = StarlarkCallable.class), + @ParamType(type = NoneType.class), + }, + doc = "An optional function applied to each element before comparison.", + defaultValue = "None") + }, + useStarlarkThread = true) + public Object min(Object key, Sequence args, StarlarkThread thread) + throws EvalException, InterruptedException { + return findExtreme( + args, + Starlark.toJavaOptional(key, StarlarkCallable.class), + Starlark.ORDERING.reverse(), + thread); + } + + @StarlarkMethod( + name = "max", + doc = + "Returns the largest one of all given arguments. If only one positional argument is" + + " provided, it must be a non-empty iterable.It is an error if elements are not" + + " comparable (for example int with string), or if no arguments are given." + + "
\n" //
+              + "max(2, 5, 4) == 5\n"
+              + "max([5, 6, 3]) == 6\n"
+              + "max(\"two\", \"three\", \"four\", key = len) ==\"three\"  # the longest\n"
+              + "max([1, -1, -2, 2], key = abs) == -2  # the first encountered with maximal key"
+              + " value\n"
+              + "
", + extraPositionals = @Param(name = "args", doc = "The elements to be checked."), + parameters = { + @Param( + name = "key", + named = true, + positional = false, + allowedTypes = { + @ParamType(type = StarlarkCallable.class), + @ParamType(type = NoneType.class), + }, + doc = "An optional function applied to each element before comparison.", + defaultValue = "None") + }, + useStarlarkThread = true) + public Object max(Object key, Sequence args, StarlarkThread thread) + throws EvalException, InterruptedException { + return findExtreme( + args, Starlark.toJavaOptional(key, StarlarkCallable.class), Starlark.ORDERING, thread); + } + + /** Returns the maximum element from this list, as determined by maxOrdering. */ + private static Object findExtreme( + Sequence args, + Optional keyFn, + Ordering maxOrdering, + StarlarkThread thread) + throws EvalException, InterruptedException { + // Args can either be a list of items to compare, or a singleton list whose element is an + // iterable of items to compare. In either case, there must be at least one item to compare. + Iterable items = (args.size() == 1) ? Starlark.toIterable(args.get(0)) : args; + try { + if (keyFn.isPresent()) { + try { + return stream(items) + .map(value -> ValueWithComparisonKey.make(value, keyFn.get(), thread)) + .max(comparing(ValueWithComparisonKey::getComparisonKey, maxOrdering)) + .get() + .getValue(); + } catch (ValueWithComparisonKey.KeyCallException ex) { + Throwables.throwIfInstanceOf(ex.getCause(), EvalException.class); + Throwables.throwIfInstanceOf(ex.getCause(), InterruptedException.class); + throw new AssertionError("Got invalid ValueWithComparisonKey.KeyCallException", ex); + } + } else { + return maxOrdering.max(items); + } + } catch (ClassCastException ex) { + throw new EvalException(ex.getMessage()); // e.g. unsupported comparison: int <=> string + } catch (NoSuchElementException ex) { + throw new EvalException("expected at least one item", ex); + } + } + + /** + * Original value decorated with its comparison key; storing the comparison key alongside the + * value ensures that we call the comparison key computation function only once per original value + * (which is important in case the function has side effects). + */ + private static final class ValueWithComparisonKey { + private final Object value; + private final Object comparisonKey; + + private ValueWithComparisonKey(Object value, Object comparisonKey) { + this.value = value; + this.comparisonKey = comparisonKey; + } + + /** + * @throws KeyCallException wrapping the exception thrown by the underlying {@link + * Starlark#fastcall} call if it threw. + */ + static ValueWithComparisonKey make( + Object value, StarlarkCallable keyFn, StarlarkThread thread) { + Object[] positional = {value}; + Object[] named = {}; + try { + return new ValueWithComparisonKey( + value, Starlark.fastcall(thread, keyFn, positional, named)); + } catch (EvalException | InterruptedException ex) { + throw new KeyCallException(ex); + } + } + + Object getValue() { + return value; + } + + Object getComparisonKey() { + return comparisonKey; + } + + /** An unchecked exception wrapping an exception thrown by {@link Starlark#fastcall}. */ + private static final class KeyCallException extends RuntimeException { + KeyCallException(Exception cause) { + super(cause); + } + } + } + + @StarlarkMethod( + name = "abs", + doc = + "Returns the absolute value of a number (a non-negative number with the same magnitude)." + + "
abs(-2.3) == 2.3
", + parameters = { + @Param( + name = "x", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = StarlarkFloat.class), + }, + doc = "A number (int or float)") + }) + public Object abs(Object x) throws EvalException { + if (x instanceof StarlarkInt starlarkInt) { + if (starlarkInt.signum() < 0) { + return StarlarkInt.uminus(starlarkInt); + } + return x; + } + + double value = ((StarlarkFloat) x).toDouble(); + return StarlarkFloat.of(Math.abs(value)); + } + + @StarlarkMethod( + name = "all", + doc = + "Returns true if all elements evaluate to True or if the collection is empty. " + + "Elements are converted to boolean using the bool function." + + "
all([\"hello\", 3, True]) == True\n"
+              + "all([-1, 0, 1]) == False
", + parameters = {@Param(name = "elements", doc = "A string or a collection of elements.")}) + public boolean all(Object collection) throws EvalException { + return !hasElementWithBooleanValue(collection, false); + } + + @StarlarkMethod( + name = "any", + doc = + "Returns true if at least one element evaluates to True. " + + "Elements are converted to boolean using the bool function." + + "
any([-1, 0, 1]) == True\n"
+              + "any([False, 0, \"\"]) == False
", + parameters = {@Param(name = "elements", doc = "A string or a collection of elements.")}) + public boolean any(Object collection) throws EvalException { + return hasElementWithBooleanValue(collection, true); + } + + private static boolean hasElementWithBooleanValue(Object seq, boolean value) + throws EvalException { + for (Object x : Starlark.toIterable(seq)) { + if (Starlark.truth(x) == value) { + return true; + } + } + return false; + } + + @StarlarkMethod( + name = "sorted", + doc = + "Returns a new sorted list containing all the elements of the supplied iterable" + + " sequence. An error may occur if any pair of elements x, y may not be compared" + + " using x < y. The elements are sorted into ascending order, unless the reverse" + + " argument is True, in which case the order is descending.\n" + + " Sorting is stable: elements that compare equal retain their original relative" + + " order.\n" // + + "
\n" //
+              + "sorted([3, 5, 4]) == [3, 4, 5]\n" //
+              + "sorted([3, 5, 4], reverse = True) == [5, 4, 3]\n" //
+              + "sorted([\"two\", \"three\", \"four\"], key = len) == [\"two\", \"four\","
+              + " \"three\"]  # sort by length\n" //
+              + "
", + parameters = { + @Param(name = "iterable", doc = "The iterable sequence to sort."), + @Param( + name = "key", + named = true, + allowedTypes = { + @ParamType(type = StarlarkCallable.class), + @ParamType(type = NoneType.class), + }, + doc = "An optional function applied to each element before comparison.", + defaultValue = "None"), + @Param( + name = "reverse", + doc = "Return results in descending order.", + named = true, + defaultValue = "False", + positional = false) + }, + useStarlarkThread = true) + public StarlarkList sorted( + StarlarkIterable iterable, Object key, boolean reverse, StarlarkThread thread) + throws EvalException, InterruptedException { + Object[] array = Starlark.toArray(iterable); + Comparator order = reverse ? Starlark.ORDERING.reversed() : Starlark.ORDERING; + + // no key? + if (key == Starlark.NONE) { + try { + Arrays.sort(array, order); + } catch (ClassCastException ex) { + throw Starlark.errorf("%s", ex.getMessage()); + } + return StarlarkList.wrap(thread.mutability(), array); + } + + // The user provided a key function. + // We must call it exactly once per element, in order, + // so use the decorate/sort/undecorate pattern. + StarlarkCallable keyfn = (StarlarkCallable) key; + + // decorate + Object[] empty = {}; + for (int i = 0; i < array.length; i++) { + Object v = array[i]; + Object k = Starlark.fastcall(thread, keyfn, new Object[] {v}, empty); + array[i] = new Object[] {k, v}; + } + + class KeyComparator implements Comparator { + EvalException e; + + @Override + public int compare(Object x, Object y) { + Object xkey = ((Object[]) x)[0]; + Object ykey = ((Object[]) y)[0]; + try { + return order.compare(xkey, ykey); + } catch (ClassCastException e) { + if (this.e == null) { + this.e = new EvalException(e.getMessage()); + } + return 0; // may cause Arrays.sort to fail; see below + } + } + } + + // sort + KeyComparator comp = new KeyComparator(); + try { + Arrays.sort(array, comp); + } catch (IllegalArgumentException unused) { + // Arrays.sort failed because comp violated the Comparator contract. + if (comp.e == null) { + // There was no exception from order.compare. + // Likely the application defined a Comparable type whose + // compareTo is not a strict weak order. + throw new IllegalStateException("sort: element ordering is not self-consistent"); + } + } + + // Sort completed, possibly with deferred errors. + if (comp.e != null) { + throw comp.e; + } + + // undecorate + for (int i = 0; i < array.length; i++) { + array[i] = ((Object[]) array[i])[1]; + } + + return StarlarkList.wrap(thread.mutability(), array); + } + + private static void reverse(Object[] array) { + for (int i = 0, j = array.length - 1; i < j; i++, j--) { + Object tmp = array[i]; + array[i] = array[j]; + array[j] = tmp; + } + } + + @StarlarkMethod( + name = "reversed", + doc = + "Returns a new, unfrozen list that contains the elements of the original iterable" + + " sequence in reversed order.
reversed([3, 5, 4]) =="
+              + " [4, 5, 3]
", + parameters = { + @Param(name = "sequence", doc = "The iterable sequence (e.g. list) to be reversed."), + }, + useStarlarkThread = true) + public StarlarkList reversed(StarlarkIterable sequence, StarlarkThread thread) + throws EvalException { + Object[] array = Starlark.toArray(sequence); + reverse(array); + return StarlarkList.wrap(thread.mutability(), array); + } + + @StarlarkMethod( + name = "tuple", + doc = + "Returns a tuple with the same elements as the given iterable value." + + "
tuple([1, 2]) == (1, 2)\n"
+              + "tuple((2, 3, 2)) == (2, 3, 2)\n"
+              + "tuple({5: \"a\", 2: \"b\", 4: \"c\"}) == (5, 2, 4)
", + parameters = {@Param(name = "x", defaultValue = "()", doc = "The object to convert.")}) + public Tuple tuple(StarlarkIterable x) throws EvalException { + if (x instanceof Tuple) { + return (Tuple) x; + } + return Tuple.wrap(Starlark.toArray(x)); + } + + @StarlarkMethod( + name = "list", + doc = + "Returns a new list with the same elements as the given iterable value." + + "
list([1, 2]) == [1, 2]\n"
+              + "list((2, 3, 2)) == [2, 3, 2]\n"
+              + "list({5: \"a\", 2: \"b\", 4: \"c\"}) == [5, 2, 4]
", + parameters = {@Param(name = "x", defaultValue = "[]", doc = "The object to convert.")}, + useStarlarkThread = true) + public StarlarkList list(StarlarkIterable x, StarlarkThread thread) throws EvalException { + return StarlarkList.wrap(thread.mutability(), Starlark.toArray(x)); + } + + @StarlarkMethod( + name = "len", + doc = + "Returns the length of a string, sequence (such as a list or tuple), dict, set, or other" + + " iterable.", + parameters = {@Param(name = "x", doc = "The value whose length to report.")}, + useStarlarkThread = true) + public int len(Object x, StarlarkThread thread) throws EvalException { + int len = Starlark.len(x); + if (len < 0) { + throw Starlark.errorf("%s is not iterable", Starlark.type(x)); + } + return len; + } + + @StarlarkMethod( + name = "str", + doc = + "Converts any object to string. This is useful for debugging." + + "
str(\"ab\") == \"ab\"\n"
+              + "str(8) == \"8\"
", + parameters = {@Param(name = "x", doc = "The object to convert.")}, + useStarlarkThread = true) + public String str(Object x, StarlarkThread thread) throws EvalException { + return Starlark.str(x, thread.getSemantics()); + } + + @StarlarkMethod( + name = "repr", + doc = + "Converts any object to a string representation. This is useful for debugging.
" + + "
repr(\"ab\") == '\"ab\"'
", + parameters = {@Param(name = "x", doc = "The object to convert.")}) + public String repr(Object x) { + return Starlark.repr(x); + } + + @StarlarkMethod( + name = "bool", + doc = + "Constructor for the bool type. " + + "It returns False if the object is None, False" + + ", an empty string (\"\"), the number 0, or an " + + "empty collection (e.g. (), []). " + + "Otherwise, it returns True.", + parameters = {@Param(name = "x", defaultValue = "False", doc = "The variable to convert.")}) + public boolean bool(Object x) throws EvalException { + return Starlark.truth(x); + } + + @StarlarkMethod( + name = "float", + doc = + "Returns x as a float value. " // + + "
  • If x is already a float, float returns it" + + " unchanged. " // + + "
  • If x is a bool, float returns 1.0 for True and 0.0" + + " for False. " // + + "
  • If x is an int, float returns the nearest" + + " finite floating-point value to x, or an error if the magnitude is too large. " // + + "
  • If x is a string, it must be a valid floating-point literal, or" + + " be equal (ignoring case) to NaN, Inf, or" + + " Infinity, optionally preceded by a + or -" + + " sign. " // + + "
" // + + "Any other value causes an error. With no argument, float() returns" + + " 0.0.", + parameters = { + @Param(name = "x", doc = "The value to convert.", defaultValue = "unbound"), + }) + public StarlarkFloat floatForStarlark(Object x) throws EvalException { + if (x instanceof String s) { + if (s.isEmpty()) { + throw Starlark.errorf("empty string"); + } + + double d; + switch (Ascii.toLowerCase(s.charAt(s.length() - 1))) { + case 'n': + case 'f': + case 'y': // {,+,-}{NaN,Inf,Infinity} + // non-finite + if (Ascii.equalsIgnoreCase(s, "nan") + || Ascii.equalsIgnoreCase(s, "+nan") + || Ascii.equalsIgnoreCase(s, "-nan")) { + d = Double.NaN; + } else if (Ascii.equalsIgnoreCase(s, "inf") + || Ascii.equalsIgnoreCase(s, "+inf") + || Ascii.equalsIgnoreCase(s, "+infinity")) { + d = Double.POSITIVE_INFINITY; + } else if (Ascii.equalsIgnoreCase(s, "-inf") || Ascii.equalsIgnoreCase(s, "-infinity")) { + d = Double.NEGATIVE_INFINITY; + } else { + throw Starlark.errorf("invalid float literal: %s", s); + } + break; + default: + // finite + try { + d = Double.parseDouble(s); + if (!Double.isFinite(d)) { + // parseDouble accepts signed "NaN" and "Infinity" (case sensitive) + // but we already handled those cases, so this indicates + // a large number rounded to infinity. + throw Starlark.errorf("floating-point number too large"); + } + } catch (NumberFormatException unused) { + throw Starlark.errorf("invalid float literal: %s", s); + } + break; + } // switch + return StarlarkFloat.of(d); + + } else if (x instanceof Boolean) { + return StarlarkFloat.of(((Boolean) x).booleanValue() ? 1 : 0); + + } else if (x instanceof StarlarkInt) { + return StarlarkFloat.of(((StarlarkInt) x).toFiniteDouble()); + + } else if (x instanceof StarlarkFloat) { + return (StarlarkFloat) x; + + } else if (x == Starlark.UNBOUND) { + return StarlarkFloat.of(0.0); + + } else { + throw Starlark.errorf("got %s, want string, int, float, or bool", Starlark.type(x)); + } + } + + @StarlarkMethod( + name = "int", + doc = + "Returns x as an int value." + + "
    " + + "
  • If x is already an int, int returns it unchanged." // + + "
  • If x is a bool, int returns 1 for True and 0 for" + + " False." // + + "
  • If x is a string, it must have the format " + + " <sign><prefix><digits>. " + + " <sign> is either \"+\", \"-\", " + + " or empty (interpreted as positive). <digits> are a " + + " sequence of digits from 0 up to base - 1, where the letters a-z " + + " (or equivalently, A-Z) are used as digits for 10-35. In the case where " + + " base is 2/8/16, <prefix> is optional and may " + + " be 0b/0o/0x (or equivalently, 0B/0O/0X) respectively; if the " + + " base is any other value besides these bases or the special value " + + " 0, the prefix must be empty. In the case where base is 0, the " + + " string is interpreted as an integer literal, in the sense that one of the " + + " bases 2/8/10/16 is chosen depending on which prefix if any is used. If " + + " base is 0, no prefix is used, and there is more than one digit, " + + " the leading digit cannot be 0; this is to avoid confusion between octal and " + + " decimal. The magnitude of the number represented by the string must be within " + + " the allowed range for the int type." // + + "
  • If x is a float, int returns the integer value of" + + " the float, rounding towards zero. It is an error if x is non-finite (NaN or" + + " infinity)." + + "
" // + + "This function fails if x is any other type, or if the value is a " + + "string not satisfying the above format. Unlike Python's int " + + "function, this function does not allow zero arguments, and does " + + "not allow extraneous whitespace for string arguments.

" // + + "Examples:

int(\"123\") == 123\n"
+              + "int(\"-123\") == -123\n"
+              + "int(\"+123\") == 123\n"
+              + "int(\"FF\", 16) == 255\n"
+              + "int(\"0xFF\", 16) == 255\n"
+              + "int(\"10\", 0) == 10\n"
+              + "int(\"-0x10\", 0) == -16\n"
+              + "int(\"-0x10\", 0) == -16\n"
+              + "int(\"123.456\") == 123\n"
+              + "
", + parameters = { + @Param(name = "x", doc = "The string to convert."), + @Param( + name = "base", + defaultValue = "unbound", + doc = + "The base used to interpret a string value; defaults to 10. Must be between 2 " + + "and 36 (inclusive), or 0 to detect the base as if x were an " + + "integer literal. This parameter must not be supplied if the value is not a " + + "string.", + named = true) + }) + public StarlarkInt intForStarlark(Object x, Object baseO) throws EvalException { + if (x instanceof String) { + int base = baseO == Starlark.UNBOUND ? 10 : Starlark.toInt(baseO, "base"); + try { + return StarlarkInt.parse((String) x, base); + } catch (NumberFormatException ex) { + throw Starlark.errorf("%s", ex.getMessage()); + } + } + + if (baseO != Starlark.UNBOUND) { + throw Starlark.errorf("can't convert non-string with explicit base"); + } + if (x instanceof Boolean) { + return StarlarkInt.of(((Boolean) x).booleanValue() ? 1 : 0); + } else if (x instanceof StarlarkInt) { + return (StarlarkInt) x; + } else if (x instanceof StarlarkFloat) { + try { + return StarlarkInt.ofFiniteDouble(((StarlarkFloat) x).toDouble()); + } catch (IllegalArgumentException unused) { + throw Starlark.errorf("can't convert float %s to int", x); + } + } + throw Starlark.errorf("got %s, want string, int, float, or bool", Starlark.type(x)); + } + + @StarlarkMethod( + name = "dict", + doc = + "Creates a dictionary from an optional positional " + + "argument and an optional set of keyword arguments. In the case where the same key " + + "is given multiple times, the last value will be used. Entries supplied via " + + "keyword arguments are considered to come after entries supplied via the " + + "positional argument.", + parameters = { + @Param( + name = "pairs", + defaultValue = "[]", + doc = "A dict, or an iterable whose elements are each of length 2 (key, value)."), + }, + extraKeywords = @Param(name = "kwargs", doc = "Dictionary of additional entries."), + useStarlarkThread = true) + public Dict dict(Object pairs, Dict kwargs, StarlarkThread thread) + throws EvalException { + // common case: dict(k=v, ...) + if (pairs instanceof StarlarkList && ((StarlarkList) pairs).isEmpty()) { + return kwargs; + } + Dict dict = Dict.of(thread.mutability()); + Dict.update("dict", dict, pairs, kwargs); + return dict; + } + + @StarlarkMethod( + name = "set", + doc = + "Experimental. This API is experimental and may change at any time. Please do not" + + " depend on it. It may be enabled on an experimental basis by setting" + + " --experimental_enable_starlark_set.\n" // + + "

Creates a new set, optionally initialized to" + + " contain the elements from a given iterable.", + parameters = { + @Param(name = "elements", defaultValue = "[]", doc = "A set, sequence, or dict."), + }, + useStarlarkThread = true) + public StarlarkSet set(Object elements, StarlarkThread thread) throws EvalException { + // Ordinarily we would use StarlarkMethod#enableOnlyWithFlag, but this doesn't work for + // top-level symbols, so enforce it here instead. + if (!thread.getSemantics().getBool(StarlarkSemantics.EXPERIMENTAL_ENABLE_STARLARK_SET)) { + throw Starlark.errorf("Use of set() requires --experimental_enable_starlark_set"); + } + return StarlarkSet.checkedCopyOf(thread.mutability(), elements); + } + + @StarlarkMethod( + name = "enumerate", + doc = + "Returns a list of pairs (two-element tuples), with the index (int) and the item from" + + " the input sequence.\n
"
+              + "enumerate([24, 21, 84]) == [(0, 24), (1, 21), (2, 84)]
\n", + parameters = { + // Note Python uses 'sequence' keyword instead of 'list'. We may want to change tihs + // some day. + @Param(name = "list", doc = "input sequence.", named = true), + @Param(name = "start", doc = "start index.", defaultValue = "0", named = true), + }, + useStarlarkThread = true) + public StarlarkList enumerate(Object input, StarlarkInt startI, StarlarkThread thread) + throws EvalException { + int start = Starlark.toInt(startI, "start"); + Object[] array = Starlark.toArray(input); + for (int i = 0; i < array.length; i++) { + array[i] = Tuple.pair(StarlarkInt.of(i + start), array[i]); // update in place + } + return StarlarkList.wrap(thread.mutability(), array); + } + + @StarlarkMethod( + name = "hash", + doc = + "Return a hash value for a string. This is computed deterministically using the same " + + "algorithm as Java's String.hashCode(), namely: " + + "
s[0] * (31^(n-1)) + s[1] * (31^(n-2)) + ... + "
+              + "s[n-1]
Hashing of values besides strings is not currently supported.", + // Deterministic hashing is important for the consistency of builds, hence why we + // promise a specific algorithm. This is in contrast to Java (Object.hashCode()) and + // Python, which promise stable hashing only within a given execution of the program. + parameters = {@Param(name = "value", doc = "String value to hash.")}) + public int hash(String value) throws EvalException { + return value.hashCode(); + } + + @StarlarkMethod( + name = "range", + doc = + "Creates a list where items go from start to stop, using a " + + "step increment. If a single argument is provided, items will " + + "range from 0 to that element." + + "
range(4) == [0, 1, 2, 3]\n"
+              + "range(3, 9, 2) == [3, 5, 7]\n"
+              + "range(3, 0, -1) == [3, 2, 1]
", + parameters = { + @Param( + name = "start_or_stop", + doc = + "Value of the start element if stop is provided, " + + "otherwise value of stop and the actual start is 0"), + @Param( + name = "stop_or_none", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "None", + doc = + "optional index of the first item not to be included in the resulting " + + "list; generation of the list stops before stop is reached."), + @Param( + name = "step", + defaultValue = "1", + doc = "The increment (default is 1). It may be negative.") + }, + useStarlarkThread = true) + public Sequence range( + StarlarkInt startOrStop, Object stopOrNone, StarlarkInt stepI, StarlarkThread thread) + throws EvalException { + int start; + int stop; + if (stopOrNone == Starlark.NONE) { + start = 0; + stop = startOrStop.toInt("stop"); + } else { + start = startOrStop.toInt("start"); + stop = Starlark.toInt(stopOrNone, "stop"); + } + int step = stepI.toInt("step"); + if (step == 0) { + throw Starlark.errorf("step cannot be 0"); + } + // TODO(adonovan): support arbitrary integers. + return new RangeList(start, stop, step); + } + + /** Returns true if the object has a field of the given name, otherwise false. */ + @StarlarkMethod( + name = "hasattr", + doc = + "Returns True if the object x has an attribute or method of the given " + + "name, otherwise False. Example:
" + + "
hasattr(ctx.attr, \"myattr\")
", + parameters = { + @Param(name = "x", doc = "The object to check."), + @Param(name = "name", doc = "The name of the attribute.") + }, + useStarlarkThread = true) + public boolean hasattr(Object obj, String name, StarlarkThread thread) throws EvalException { + return Starlark.hasattr(thread.getSemantics(), obj, name); + } + + @StarlarkMethod( + name = "getattr", + doc = + "Returns the struct's field of the given name if it exists. If not, it either returns " + + "default (if specified) or raises an error. " + + "getattr(x, \"foobar\") is equivalent to x.foobar." + + "
getattr(ctx.attr, \"myattr\")\n"
+              + "getattr(ctx.attr, \"myattr\", \"mydefault\")
", + parameters = { + @Param(name = "x", doc = "The struct whose attribute is accessed."), + @Param(name = "name", doc = "The name of the struct attribute."), + @Param( + name = "default", + defaultValue = "unbound", + doc = + "The default value to return in case the struct " + + "doesn't have an attribute of the given name.") + }, + useStarlarkThread = true) + public Object getattr(Object obj, String name, Object defaultValue, StarlarkThread thread) + throws EvalException, InterruptedException { + return Starlark.getattr( + thread.mutability(), + thread.getSemantics(), + obj, + name, + defaultValue == Starlark.UNBOUND ? null : defaultValue); + } + + @StarlarkMethod( + name = "dir", + doc = + "Returns a list of strings: the names of the attributes and " + + "methods of the parameter object.", + parameters = {@Param(name = "x", doc = "The object to check.")}, + useStarlarkThread = true) + public StarlarkList dir(Object object, StarlarkThread thread) throws EvalException { + return Starlark.dir(thread.mutability(), thread.getSemantics(), object); + } + + @StarlarkMethod( + name = "fail", + doc = "Causes execution to fail with an error.", + parameters = { + // TODO(adonovan): remove. See https://github.com/bazelbuild/starlark/issues/47. + @Param( + name = "msg", + doc = + "Deprecated: use positional arguments instead. " + + "This argument acts like an implicit leading positional argument.", + defaultValue = "None", + positional = false, + named = true), + // TODO(adonovan): remove. See https://github.com/bazelbuild/starlark/issues/47. + @Param( + name = "attr", + allowedTypes = { + @ParamType(type = String.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "None", + doc = + "Deprecated. Causes an optional prefix containing this string to be added to the" + + " error message.", + positional = false, + named = true), + @Param( + name = "sep", + defaultValue = "\" \"", + named = true, + positional = false, + doc = "The separator string between the objects, default is space (\" \").") + }, + extraPositionals = + @Param( + name = "args", + doc = + "A list of values, formatted with debugPrint (which is equivalent to str by" + + " default) and joined with sep (defaults to \" \"), that appear in the" + + " error message."), + useStarlarkThread = true) + public void fail(Object msg, Object attr, String sep, Tuple args, StarlarkThread thread) + throws EvalException { + Printer printer = new Printer(); + boolean needSeparator = false; + if (attr != Starlark.NONE) { + printer.append("attribute ").append((String) attr).append(":"); + needSeparator = true; + } + // msg acts like a leading element of args. + if (msg != Starlark.NONE) { + if (needSeparator) { + printer.append(sep); + } + printer.debugPrint(msg, thread); + needSeparator = true; + } + for (Object arg : args) { + if (needSeparator) { + printer.append(sep); + } + printer.debugPrint(arg, thread); + needSeparator = true; + } + throw Starlark.errorf("%s", printer.toString()); + } + + @StarlarkMethod( + name = "print", + doc = + "Prints args as debug output. It will be prefixed with the string " + + "\"DEBUG\" and the location (file and line number) of this call. The " + + "exact way in which the arguments are converted to strings is unspecified and may " + + "change at any time. In particular, it may be different from (and more detailed " + + "than) the formatting done by str() and repr()." + + "

Using print in production code is discouraged due to the spam it " + + "creates for users. For deprecations, prefer a hard error using " + + "fail() whenever possible.", + parameters = { + @Param( + name = "sep", + defaultValue = "\" \"", + named = true, + positional = false, + doc = "The separator string between the objects, default is space (\" \").") + }, + // NB: as compared to Python3, we're missing optional named-only arguments 'end' and 'file' + extraPositionals = @Param(name = "args", doc = "The objects to print."), + useStarlarkThread = true) + public void print(String sep, Sequence args, StarlarkThread thread) throws EvalException { + Printer p = new Printer(); + String separator = ""; + for (Object x : args) { + p.append(separator); + p.debugPrint(x, thread); + separator = sep; + } + // The PRINT_TEST_MARKER key is used in tests to verify the effects of command-line options. + // See starlark_flag_test.sh, which runs bazel with --internal_starlark_flag_test_canary. + if (thread.getSemantics().getBool(StarlarkSemantics.PRINT_TEST_MARKER)) { + p.append("<== Starlark flag test ==>"); + } + + thread.getPrintHandler().print(thread, p.toString()); + } + + @StarlarkMethod( + name = "type", + doc = + "Returns the type name of its argument. This is useful for debugging and " + + "type-checking. Examples:" + + "

"
+              + "type(2) == \"int\"\n"
+              + "type([1]) == \"list\"\n"
+              + "type(struct(a = 2)) == \"struct\""
+              + "
" + + "This function might change in the future. To write Python-compatible code and " + + "be future-proof, use it only to compare return values: " + + "
"
+              + "if type(x) == type([]):  # if x is a list"
+              + "
", + parameters = {@Param(name = "x", doc = "The object to check type of.")}) + public String type(Object object) { + // There is no 'type' type in Starlark, so we return a string with the type name. + return Starlark.type(object); + } + + @StarlarkMethod( + name = "zip", + doc = + "Returns a list of tuples, where the i-th tuple contains " + + "the i-th element from each of the argument sequences or iterables. The list has " + + "the size of the shortest input. With a single iterable argument, it returns a " + + "list of 1-tuples. With no arguments, it returns an empty list. Examples:" + + "
"
+              + "zip()  # == []\n"
+              + "zip([1, 2])  # == [(1,), (2,)]\n"
+              + "zip([1, 2], [3, 4])  # == [(1, 3), (2, 4)]\n"
+              + "zip([1, 2], [3, 4, 5])  # == [(1, 3), (2, 4)]
", + extraPositionals = @Param(name = "args", doc = "lists to zip."), + useStarlarkThread = true) + public StarlarkList zip(Sequence args, StarlarkThread thread) throws EvalException { + StarlarkList result = StarlarkList.newList(thread.mutability()); + int ncols = args.size(); + if (ncols > 0) { + Iterator[] iterators = new Iterator[ncols]; + for (int i = 0; i < ncols; i++) { + iterators[i] = Starlark.toIterable(args.get(i)).iterator(); + } + rows: + for (; ; ) { + Object[] elem = new Object[ncols]; + for (int i = 0; i < ncols; i++) { + Iterator it = iterators[i]; + if (!it.hasNext()) { + break rows; + } + elem[i] = it.next(); + } + result.addElement(Tuple.wrap(elem)); + } + } + return result; + } + + /** Starlark bool type. */ + @StarlarkBuiltin( + name = "bool", + category = "core", + doc = + "A type to represent booleans. There are only two possible values: " + + "True and False. Any value can be converted to a boolean using the " + + "bool function.") + static final class BoolModule implements StarlarkValue {} // (documentation only) +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Module.java b/third_party/bazel/main/java/net/starlark/java/eval/Module.java new file mode 100644 index 000000000..b7279a202 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/Module.java @@ -0,0 +1,321 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nullable; +import net.starlark.java.syntax.Resolver; + +/** + * A {@link Module} represents a Starlark module, a container of global variables populated by + * executing a Starlark file. Each top-level assignment updates a global variable in the module. + * + *

Each module references its "predeclared" environment, which is often shared among many + * modules. These are the names that are defined even at the start of execution. For example, in + * Bazel, the predeclared environment of the module for a BUILD or .bzl file defines name values + * such as cc_binary and glob. + * + *

The predeclared environment implicitly includes the "universal" names present in every + * Starlark thread in every dialect, such as None, len, and str; see {@link Starlark#UNIVERSE}. + * + *

Global bindings in a Module may shadow bindings inherited from the predeclared block. + * + *

A module may carry an arbitrary piece of client data. In Bazel, for example, the client data + * records the module's build label (such as "//dir:file.bzl"). This client data is accessible to + * (for instance) application-defined builtin methods. + * + *

You may create a Module using {@link #create}, {@link #withPredeclared}, or {@link + * #withPredeclaredAndData}. The latter two give you the ability to add predeclared bindings (beyond + * the universal ones) and client data. The particular {@link StarlarkSemantics} and client data may + * filter what predeclared bindings are available via {@link GuardedValue}. + */ +public final class Module implements Resolver.Module { + + // The module's predeclared environment. Excludes UNIVERSE bindings. Values that are conditionally + // present are stored as GuardedValues regardless of whether they are actually enabled. + private final ImmutableMap predeclared; + + // The module's global variables, in order of creation. + private final LinkedHashMap globalIndex = new LinkedHashMap<>(); + private Object[] globals = new Object[8]; + + // An optional piece of application-specific metadata associated with the module/file. + // Its toString appears to Starlark in str(function): "". + @Nullable private final Object clientData; + + private final StarlarkSemantics semantics; + + // An optional doc string for the module. Set after construction when evaluating a .bzl file. + @Nullable private String documentation; + + private Module( + ImmutableMap predeclared, + @Nullable Object clientData, + StarlarkSemantics semantics) { + this.predeclared = predeclared; + this.clientData = clientData; + this.semantics = semantics; + } + + /** + * Constructs a Module with the specified predeclared bindings (filtered by the semantics), in * + * addition to the standard environment, {@link Starlark#UNIVERSE}. No client data is set. + */ + public static Module withPredeclared( + StarlarkSemantics semantics, Map predeclared) { + return withPredeclaredAndData(semantics, predeclared, null); + } + + /** + * Constructs a Module as above, but with the specified client data -- an arbitrary + * application-specific value to be associated with this Module. Client data may also affect the + * filtering of predeclareds alongside the semantics. + */ + public static Module withPredeclaredAndData( + StarlarkSemantics semantics, Map predeclared, @Nullable Object clientData) { + return new Module(ImmutableMap.copyOf(predeclared), clientData, semantics); + } + + /** + * Creates a module with no predeclared bindings other than the standard environment, {@link + * Starlark#UNIVERSE}, and with no client data. + */ + public static Module create() { + return new Module( + /* predeclared= */ ImmutableMap.of(), /* clientData= */ null, StarlarkSemantics.DEFAULT); + } + + /** + * Returns the module (file) of the {@code depth}-th innermost enclosing Starlark function on the + * call stack, or null if number of the active calls that are functions defined in Starlark is + * less than or equal to {@code depth}. + * + *

This method is a temporary workaround for Starlarkification, to check {@code _builtin} + * restriction and should not be used anywhere else. + * + * @param depth the depth for the callstack. + * @throws IllegalArgumentException if {@code depth} is negative. + */ + @Nullable + public static Module ofInnermostEnclosingStarlarkFunction(StarlarkThread thread, int depth) { + StarlarkFunction fn = thread.getInnermostEnclosingStarlarkFunction(depth); + if (fn != null) { + return fn.getModule(); + } + return null; + } + + /** + * Returns the module (file) of the innermost enclosing Starlark function on the call stack, or + * null if none of the active calls are functions defined in Starlark. + * + *

The name of this function is intentionally horrible to make you feel bad for using it. + */ + @Nullable + public static Module ofInnermostEnclosingStarlarkFunction(StarlarkThread thread) { + return ofInnermostEnclosingStarlarkFunction(thread, 0); + } + + /** + * Replaces an enabled {@link GuardedValue} with the value it guards. + * + *

A disabled {@link GuardedValue} is left in place for error reporting upon access, and should + * be treated as unavailable. + */ + private Object filterGuardedValue(Object v) { + Preconditions.checkNotNull(v); + if (!(v instanceof GuardedValue)) { + return v; + } + GuardedValue gv = (GuardedValue) v; + return gv.isObjectAccessibleUsingSemantics(semantics, clientData) ? gv.getObject() : gv; + } + + /** Returns the client data associated with this module. */ + @Nullable + public Object getClientData() { + return clientData; + } + + /** Sets the module's doc string. It may be retrieved using {@link #getDocumentation}. */ + public void setDocumentation(String documentation) { + this.documentation = documentation; + } + + /** + * Returns the module's doc string, or null if absent. + * + *

Morally equivalent to calling {@code program.getResolvedFunction().getDocumentation()} when + * the Module has a corresponding {@link net.starlark.java.syntax.Program}. We need to separately + * save the doc string inside the Module because (1) a Module will usually outlive the Program; + * and (2) there isn't always a 1-to-1 match between a Module and a Program (multiple programs may + * be executed in the same module in REPL or in tests). + */ + @Nullable + public String getDocumentation() { + return documentation; + } + + /** + * Returns the value of a predeclared (not universal) binding in this module. + * + *

In the case that the predeclared is a {@link GuardedValue}: If it is enabled, the underlying + * value is returned, otherwise the {@code GuardedValue} itself is returned for error reporting. + */ + @Nullable + public Object getPredeclared(String name) { + Object value = predeclared.get(name); + if (value == null) { + return null; + } + return filterGuardedValue(value); + } + + /** + * Returns this module's additional predeclared bindings. (Excludes {@link Starlark#UNIVERSE}.) + * + *

The map reflects any filtering of {@link GuardedValue}: enabled ones are replaced by the + * underlying values that they guard, while disabled ones are left in place for error reporting. + */ + public Map getPredeclaredBindings() { + return Maps.transformValues(predeclared, this::filterGuardedValue); + } + + /** + * Returns an immutable mapping containing the global variables of this module. + * + *

The bindings are returned in a deterministic order (for a given sequence of initial values + * and updates). + */ + public ImmutableMap getGlobals() { + int n = globalIndex.size(); + ImmutableMap.Builder m = ImmutableMap.builderWithExpectedSize(n); + for (Map.Entry e : globalIndex.entrySet()) { + Object v = getGlobalByIndex(e.getValue()); + if (v != null) { + m.put(e.getKey(), v); + } + } + return m.buildOrThrow(); + } + + /** Implements the resolver's module interface. */ + @Override + public Resolver.Scope resolve(String name) throws Undefined { + // global? + if (globalIndex.containsKey(name)) { + return Resolver.Scope.GLOBAL; + } + + // predeclared? + Object v = getPredeclared(name); + if (v != null) { + if (v instanceof GuardedValue) { + // Name is correctly spelled, but access is disabled by a flag or by client data. + throw new Undefined( + ((GuardedValue) v).getErrorFromAttemptingAccess(name), /*candidates=*/ null); + } + return Resolver.Scope.PREDECLARED; + } + + // universal? + if (Starlark.UNIVERSE.containsKey(name)) { + return Resolver.Scope.UNIVERSAL; + } + + // undefined + Set candidates = new HashSet<>(); + candidates.addAll(globalIndex.keySet()); + candidates.addAll(predeclared.keySet()); + candidates.addAll(Starlark.UNIVERSE.keySet()); + throw new Undefined(String.format("name '%s' is not defined", name), candidates); + } + + /** + * Returns the value of the specified global variable, or null if not bound. Does not look in the + * predeclared environment. + */ + @Nullable + public Object getGlobal(String name) { + Integer i = globalIndex.get(name); + return i != null ? globals[i] : null; + } + + /** + * Sets the value of a global variable based on its index in this module ({@see + * getIndexOfGlobal}). + */ + void setGlobalByIndex(int i, Object v) { + Preconditions.checkArgument(i < globalIndex.size()); + this.globals[i] = v; + } + + /** + * Returns the value of a global variable based on its index in this module ({@see + * getIndexOfGlobal}.) Returns null if the variable has not been assigned a value. + */ + @Nullable + Object getGlobalByIndex(int i) { + Preconditions.checkArgument(i < globalIndex.size()); + return this.globals[i]; + } + + /** + * Returns the index within this Module of a global variable, given its name, creating a new slot + * for it if needed. The numbering of globals used by these functions is not the same as the + * numbering within any compiled Program. Thus each StarlarkFunction must contain a secondary + * index mapping Program indices (from Binding.index) to Module indices. + */ + int getIndexOfGlobal(String name) { + int i = globalIndex.size(); + Integer prev = globalIndex.putIfAbsent(name, i); + if (prev != null) { + return prev; + } + if (i == globals.length) { + globals = Arrays.copyOf(globals, globals.length << 1); // grow by doubling + } + return i; + } + + /** Returns a list of indices of a list of globals; {@see getIndexOfGlobal}. */ + int[] getIndicesOfGlobals(List globals) { + int n = globals.size(); + int[] array = new int[n]; + for (int i = 0; i < n; i++) { + array[i] = getIndexOfGlobal(globals.get(i)); + } + return array; + } + + /** Updates a global binding in the module environment. */ + public void setGlobal(String name, Object value) { + Preconditions.checkNotNull(value, "Module.setGlobal(%s, null)", name); + setGlobalByIndex(getIndexOfGlobal(name), value); + } + + @Override + public String toString() { + return String.format("", clientData == null ? "?" : clientData); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Mutability.java b/third_party/bazel/main/java/net/starlark/java/eval/Mutability.java new file mode 100644 index 000000000..586dd5e95 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/Mutability.java @@ -0,0 +1,281 @@ +// Copyright 2015 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.eval; + +import com.google.common.base.Joiner; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.util.IdentityHashMap; + +/** + * An object that manages the capability to mutate Starlark objects and their {@link + * StarlarkThread}s. Collectively, the managed objects are called {@link Freezable}s. + * + *

Each {@code StarlarkThread}, and each of the mutable Starlark values that are created in that + * {@code StarlarkThread}, holds a pointer to the same {@code Mutability} instance. Once the {@code + * StarlarkThread} is done evaluating, its {@code Mutability} is irreversibly closed ("frozen"). At + * that point, it is no longer possible to change either the bindings in that {@code StarlarkThread} + * or the state of its objects. This protects each {@code StarlarkThread} from unintentional and + * unsafe modification. + * + *

{@code Mutability}s enforce isolation between {@code StarlarkThread}s; it is illegal for an + * evaluation in one {@code StarlarkThread} to affect the bindings or values of another. In + * particular, the {@code StarlarkThread} for any Starlark module is frozen before its symbols can + * be imported for use by another module. Each individual {@code StarlarkThread}'s evaluation is + * single-threaded, so this isolation also translates to thread safety. Any number of threads may + * simultaneously access frozen data. (The {@code Mutability} itself is also thread-safe if and only + * if it is frozen.} + * + *

Although the mutability pointer of a {@code Freezable} contains some debugging information + * about its context, this should not affect the {@code Freezable}'s semantics. From a behavioral + * point of view, the only thing that matters is whether the {@code Mutability} is frozen, not what + * particular {@code Mutability} object is pointed to. + * + *

When a Starlark program iterates over a mutable sequence value in a for-loop or comprehension, + * the sequence value becomes temporarily immutable for the duration of the loop. Conceptually, the + * value maintains a counter of active iterations, and the interpreter notifies the {@code + * Freezable} value before and after the loop so that it can alter its counter by calling its {@code + * updateIteratorCount} method. While the counter value is nonzero, the value should cause all + * attempts to mutate it to fail. The temporary immutability applies only to the sequence itself, + * not to its elements. Once a mutable sequence becomes frozen, there is no need to count active + * iterators (and doing so would be racy as frozen objects may be published to other Starlark + * threads). The default implementation of {@code updateIteratorCount} uses a set of counters in the + * Mutability, but a Freezable object may define a more efficient intrusive counter implementation. + * + *

We follow two disciplines to ensure safety. First, all mutation methods of a Freezable value + * must confirm that the value's Mutability is not yet frozen, nor is the value temporarily + * immutable due to active iterators. + * + *

Second, {@code Mutability}s are created using the try-with-resource style: + * + * 

{@code
+ * try (Mutability mutability = Mutability.create(name, ...)) { ... }
+ * }
+ * + * The general pattern is to create a {@code Mutability}, build an {@code StarlarkThread}, mutate + * that {@code StarlarkThread} and its objects, and possibly return the result from within the + * {@code try} block, relying on the try-with-resource construct to ensure that everything gets + * frozen before the result is used. The only code that should create a {@code Mutability} without + * using try-with-resource is test code that is not part of the Bazel jar. + * + *

We keep some (unchecked) invariants regarding where {@code Mutability} objects may appear + * within a compound value. + * + *

    + *
  1. A compound value can never contain an unfrozen {@code Mutability} for any {@code + * StarlarkThread} except the one currently being evaluated. + *
  2. If a value has the special {@link #IMMUTABLE} {@code Mutability}, all of its contents are + * themselves deeply immutable too (i.e. have frozen {@code Mutability}s). + *
+ * + * It follows that, if these invariants hold, an unfrozen value cannot appear as the child of a + * value whose {@code Mutability} is already frozen. + * + *

There is a special API for freezing individual values rather than whole {@code + * StarlarkThread}s. Because this API makes it easier to violate the above invariants, you should + * avoid using it if at all possible; at the moment it is only used for serialization. Under this + * API, you may call {@link Freezable#unsafeShallowFreeze} to reset a value's {@code Mutability} + * pointer to be {@link #IMMUTABLE}. This operation has no effect on the {@code Mutability} itself. + * It is up to the caller to preserve or restore the above invariants by ensuring that any deeply + * contained values are also frozen. For safety and explicitness, this operation is disallowed + * unless the {@code Mutability}'s {@link #allowsUnsafeShallowFreeze} method returns true. + */ +public final class Mutability implements AutoCloseable { + + // Maps each temporarily frozen Freezable value to the (positive) count of active iterators over + // the value. This field is set to null when the Mutability becomes permanently frozen, at which + // point there is no need to track iterators. This map does not contain Freezable values that + // define their own implementation of updateIteratorCount. + private IdentityHashMap iteratorCount = + new IdentityHashMap<>(10); // 10 nested for-loops seems plenty + + // An optional list of values that are formatted with toString and joined with spaces to yield the + // "annotation", an internal name describing the purpose of this Mutability. + private final Object[] annotation; + + /** Controls access to {@link Freezable#unsafeShallowFreeze}. */ + private final boolean allowsUnsafeShallowFreeze; + + private Mutability(Object[] annotation, boolean allowsUnsafeShallowFreeze) { + this.annotation = annotation; + this.allowsUnsafeShallowFreeze = allowsUnsafeShallowFreeze; + } + + /** + * Creates a {@code Mutability}. + * + * @param annotation a list of objects whose toString representations are joined with spaces to + * yield the annotation, an internal name describing the purpose of this Mutability. + */ + public static Mutability create(Object... annotation) { + return new Mutability(annotation, /*allowsUnsafeShallowFreeze=*/ false); + } + + /** + * Creates a {@code Mutability} whose objects can be individually frozen; see docstrings for + * {@link Mutability} and {@link Freezable#unsafeShallowFreeze}. + */ + public static Mutability createAllowingShallowFreeze(Object... annotation) { + return new Mutability(annotation, /*allowsUnsafeShallowFreeze=*/ true); + } + + /** Returns the Mutability's "annotation", an internal name describing its purpose. */ + public String getAnnotation() { + // The annotation string is computed when needed, typically never, + // to avoid the performance penalty of materializing it eagerly. + return Joiner.on(" ").join(annotation); + } + + @Override + public String toString() { + return (isFrozen() ? "(" : "[") + getAnnotation() + (isFrozen() ? ")" : "]"); + } + + public boolean isFrozen() { + return this.iteratorCount == null; + } + + // Defines the default behavior of mutable Freezable sequence values, + // which become temporarily immutable while there are active iterators. + private boolean updateIteratorCount(Freezable x, int delta) { + if (isFrozen()) { + return false; + } + int i = this.iteratorCount.getOrDefault(x, 0); + if (delta > 0) { + i++; + this.iteratorCount.put(x, i); + } else if (delta < 0) { + i--; + if (i == 0) { + this.iteratorCount.remove(x); + } else if (i > 0) { + this.iteratorCount.put(x, i); + } else { + throw new IllegalStateException("zero value in this.iteratorCount"); + } + } + return i > 0; + } + + /** + * Freezes this {@code Mutability}, rendering all {@link Freezable} objects that refer to it + * immutable. + * + *

Note that freezing does not directly touch all the {@code Freezables}, so this operation is + * constant-time. + * + * @return this object, in the fluent style + */ + @CanIgnoreReturnValue + public Mutability freeze() { + this.iteratorCount = null; + return this; + } + + @Override + public void close() { + freeze(); + } + + /** + * Returns whether {@link Freezable}s having this {@code Mutability} allow the {@link + * #unsafeShallowFreeze} operation. + */ + public boolean allowsUnsafeShallowFreeze() { + return allowsUnsafeShallowFreeze; + } + + /** + * An object that refers to a {@link Mutability} to decide whether to allow mutation. All {@link + * Freezable} Starlark objects created within a given {@link StarlarkThread} will share the same + * {@code Mutability} as that {@code StarlarkThread}. + */ + public interface Freezable { + /** + * Returns the {@link Mutability} associated with this {@code Freezable}. This should not change + * over the lifetime of the object, except by calling {@link #unsafeShallowFreeze} if + * applicable. + */ + Mutability mutability(); + + /** + * Registers a change to this Freezable's iterator count and reports whether it is temporarily + * immutable. + * + *

If the value is permanently frozen ({@code mutability().isFrozen()), this function is a + * no-op that returns false. + * + *

Otherwise, if delta is positive, this increments the count of active iterators over the + * value, causing it to appear temporarily frozen (if it wasn't already). If delta is negative, + * the counter is decremented, and if delta is zero the counter is unchanged. It is illegal to + * decrement the counter if it was already zero. The return value is true if the count is + * positive after the change, and false otherwise. + * + *

The default implementation stores the counter of iterators in a hash table in the + * Mutability, but a subclass of Freezable may define a more efficient implementation such as an + * integer field in the freezable value itself. + * + *

Call this function with a positive value when starting an iteration and with a negative + * value when ending it. + */ + default boolean updateIteratorCount(int delta) { + return mutability().updateIteratorCount(this, delta); + } + + /** + * Freezes this object (and not its contents). Use with care. + * + *

This method is optional (i.e. may throw {@link UnsupportedOperationException}). + * + *

If this object's {@link Mutability} is 1) not frozen, and 2) has {@link + * #allowsUnsafeShallowFreeze} return true, then the object's {@code Mutability} reference is + * updated to point to {@link #IMMUTABLE}. Otherwise, this method throws {@link + * IllegalArgumentException}. + * + *

It is up to the caller to ensure that any contents of this {@code Freezable} are also + * frozen in order to preserve/restore the invariant that an immutable value cannot contain a + * mutable one. Note that thread-safety is not guaranteed otherwise. + */ + default void unsafeShallowFreeze() { + throw new UnsupportedOperationException(); + } + + /** + * Throws {@link IllegalArgumentException} if the precondition for {@link #unsafeShallowFreeze} + * is violated. To be used by implementors of {@link #unsafeShallowFreeze}. + */ + static void checkUnsafeShallowFreezePrecondition( + Freezable freezable) { + Mutability mutability = freezable.mutability(); + if (mutability.isFrozen()) { + // It's not safe to rewrite the Mutability pointer if this is already frozen, because we + // could be accessed by multiple threads. + throw new IllegalArgumentException( + "cannot call unsafeShallowFreeze() on an object whose Mutability is already frozen"); + } + if (!mutability.allowsUnsafeShallowFreeze()) { + throw new IllegalArgumentException( + "cannot call unsafeShallowFreeze() on a mutable object whose Mutability's " + + "allowsUnsafeShallowFreeze() == false"); + } + } + } + + /** + * A {@code Mutability} indicating that a value is deeply immutable. + * + *

It is not associated with any particular {@link StarlarkThread}. + */ + public static final Mutability IMMUTABLE = create("IMMUTABLE").freeze(); +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/MutableStarlarkList.java b/third_party/bazel/main/java/net/starlark/java/eval/MutableStarlarkList.java new file mode 100644 index 000000000..339814191 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/MutableStarlarkList.java @@ -0,0 +1,230 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import java.util.Arrays; +import java.util.Collection; +import javax.annotation.Nullable; + +/** A mutable implementation of StarlarkList. */ +final class MutableStarlarkList extends StarlarkList { + + // The implementation strategy is similar to ArrayList, + // but without the extra indirection of using ArrayList. + + // elems[0:size] holds the logical elements, and elems[size:] are not used. + // elems.getClass() == Object[].class. This is necessary to avoid ArrayStoreException. + private int size; + private int iteratorCount; // number of active iterators (unused once frozen) + Object[] elems = StarlarkList.EMPTY_ARRAY; // elems[i] == null iff i >= size + + /** Final except for {@link #unsafeShallowFreeze}; must not be modified any other way. */ + private Mutability mutability; + + MutableStarlarkList(@Nullable Mutability mutability, Object[] elems, int size) { + Preconditions.checkArgument(elems.getClass() == Object[].class); + this.elems = elems; + this.size = size; + this.mutability = mutability == null ? Mutability.IMMUTABLE : mutability; + } + + @Override + public boolean isImmutable() { + return mutability().isFrozen(); + } + + @Override + public boolean updateIteratorCount(int delta) { + if (mutability().isFrozen()) { + return false; + } + if (delta > 0) { + iteratorCount++; + } else if (delta < 0) { + iteratorCount--; + } + return iteratorCount > 0; + } + + @Override + public Mutability mutability() { + return mutability; + } + + @Override + public void unsafeShallowFreeze() { + Mutability.Freezable.checkUnsafeShallowFreezePrecondition(this); + this.mutability = Mutability.IMMUTABLE; + } + + @Override + public ImmutableList getImmutableList() { + // Optimization: a frozen array needn't be copied. + // If the entire array is full, we can wrap it directly. + if (elems.length == size && mutability().isFrozen()) { + return Tuple.wrapImmutable(elems); + } + + return ImmutableList.copyOf(this); + } + + @Override + @SuppressWarnings("unchecked") + public E get(int i) { + Preconditions.checkElementIndex(i, size); + return (E) elems[i]; // unchecked + } + + @Override + public int size() { + return size; + } + + @Override + public boolean contains(Object o) { + // StarlarkList contains only valid Starlark objects (which are non-null) + if (o == null) { + return false; + } + for (int i = 0; i < size; i++) { + Object elem = elems[i]; + if (o.equals(elem)) { + return true; + } + } + return false; + } + + // Postcondition: elems.length >= mincap. + private void grow(int mincap) { + int oldcap = elems.length; + if (oldcap < mincap) { + int newcap = oldcap + (oldcap >> 1); // grow by at least 50% + if (newcap < mincap) { + newcap = mincap; + } + elems = Arrays.copyOf(elems, newcap); + } + } + + // Grow capacity enough to insert given number of elements + private void growAdditional(int additional) throws EvalException { + int mincap = addSizesAndFailIfExcessive(size, additional); + grow(mincap); + } + + @Override + public void addElement(E element) throws EvalException { + Starlark.checkMutable(this); + growAdditional(1); + elems[size++] = element; + } + + @Override + public void addElementAt(int index, E element) throws EvalException { + Starlark.checkMutable(this); + growAdditional(1); + System.arraycopy(elems, index, elems, index + 1, size - index); + elems[index] = element; + size++; + } + + @Override + public void addElements(Iterable elements) throws EvalException { + Starlark.checkMutable(this); + if (elements instanceof MutableStarlarkList that) { + // (safe even if this == that) + growAdditional(that.size); + System.arraycopy(that.elems, 0, this.elems, this.size, that.size); + this.size += that.size; + } else if (elements instanceof Collection that) { + // collection of known size + growAdditional(that.size()); + for (Object x : that) { + elems[size++] = x; + } + } else { + // iterable + for (Object x : elements) { + growAdditional(1); + elems[size++] = x; + } + } + } + + @Override + public void removeElementAt(int index) throws EvalException { + Starlark.checkMutable(this); + int n = size - index - 1; + if (n > 0) { + System.arraycopy(elems, index + 1, elems, index, n); + } + elems[--size] = null; // aid GC + } + + @Override + public void setElementAt(int index, E value) throws EvalException { + Starlark.checkMutable(this); + Preconditions.checkArgument(index < size); + elems[index] = value; + } + + @Override + public void clearElements() throws EvalException { + Starlark.checkMutable(this); + for (int i = 0; i < size; i++) { + elems[i] = null; // aid GC + } + size = 0; + } + + /** Returns a new array of class Object[] containing the list elements. */ + @Override + public Object[] toArray() { + return size != 0 ? Arrays.copyOf(elems, size, Object[].class) : EMPTY_ARRAY; + } + + @SuppressWarnings("unchecked") + @Override + public T[] toArray(T[] a) { + if (a.length < size) { + return (T[]) Arrays.copyOf(elems, size, a.getClass()); + } else { + System.arraycopy(elems, 0, a, 0, size); + Arrays.fill(a, size, a.length, null); + return a; + } + } + + @Override + Object[] elems() { + return elems; + } + + @Override + public StarlarkList unsafeOptimizeMemoryLayout() { + Preconditions.checkState(mutability.isFrozen()); + // Canonicalize our Mutability, on the off-chance the old one may be freed. + mutability = Mutability.IMMUTABLE; + if (elems.length > size) { + // shrink the Object array + elems = Arrays.copyOf(elems, size); + } + // Give the caller an immutable specialization of StarlarkList. + return wrap(mutability, elems); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/NoneType.java b/third_party/bazel/main/java/net/starlark/java/eval/NoneType.java new file mode 100644 index 000000000..7aca052f3 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/NoneType.java @@ -0,0 +1,51 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import javax.annotation.concurrent.Immutable; +import net.starlark.java.annot.StarlarkBuiltin; + +/** The type of the Starlark None value. */ +@StarlarkBuiltin( + name = "NoneType", + documented = false, + doc = "The type of the Starlark None value.") +@Immutable +public final class NoneType implements StarlarkValue { + + static final NoneType NONE = new NoneType(); + + private NoneType() {} + + @Override + public String toString() { + return "None"; + } + + @Override + public boolean isImmutable() { + return true; + } + + @Override + public boolean truth() { + return false; + } + + @Override + public void repr(Printer printer) { + printer.append("None"); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/ParamDescriptor.java b/third_party/bazel/main/java/net/starlark/java/eval/ParamDescriptor.java new file mode 100644 index 000000000..5f9852e2a --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/ParamDescriptor.java @@ -0,0 +1,247 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nullable; +import net.starlark.java.annot.Param; +import net.starlark.java.annot.ParamType; +import net.starlark.java.syntax.FileOptions; +import net.starlark.java.syntax.ParserInput; +import net.starlark.java.syntax.SyntaxError; + +/** A value class for storing {@link Param} metadata to avoid using Java proxies. */ +final class ParamDescriptor { + + private final String name; + @Nullable private final Object defaultValue; + private final boolean named; + private final boolean positional; + // Null means any class is allowed. + // Should be not empty otherwise. + @Nullable private final List> allowedClasses; + // The semantics flag responsible for disabling this parameter, or null if enabled. + // It is an error for Starlark code to supply a value to a disabled parameter. + @Nullable private final String disabledByFlag; + + private ParamDescriptor( + String name, + String defaultExpr, + boolean named, + boolean positional, + List> allowedClasses, + @Nullable String disabledByFlag) { + this.name = name; + // TODO(adonovan): apply the same validation logic to the default value + // as we do to caller-supplied values (see BuiltinFunction.checkParamValue). + this.defaultValue = defaultExpr.isEmpty() ? null : evalDefault(name, defaultExpr); + this.named = named; + this.positional = positional; + if (allowedClasses.contains(Object.class)) { + this.allowedClasses = null; + } else { + this.allowedClasses = allowedClasses; + } + this.disabledByFlag = disabledByFlag; + } + + /** + * Returns a {@link ParamDescriptor} representing the given raw {@link Param} annotation and the + * given semantics. + */ + static ParamDescriptor of(Param param, Class paramClass, StarlarkSemantics starlarkSemantics) { + String defaultExpr = param.defaultValue(); + String disabledByFlag = null; + if (!starlarkSemantics.isFeatureEnabledBasedOnTogglingFlags( + param.enableOnlyWithFlag(), param.disableWithFlag())) { + defaultExpr = param.valueWhenDisabled(); + disabledByFlag = + !param.enableOnlyWithFlag().isEmpty() + ? param.enableOnlyWithFlag() + : param.disableWithFlag(); + Preconditions.checkState(!disabledByFlag.isEmpty()); + } + + // Compute set of allowed classes. + ParamType[] allowedTypes = param.allowedTypes(); + List> allowedClasses = new ArrayList<>(); + if (allowedTypes.length > 0) { + for (ParamType pt : allowedTypes) { + allowedClasses.add(pt.type()); + } + } else { + // Use the class of the parameter itself. + // Interpret primitive boolean parameter as j.l.Boolean. + allowedClasses.add(paramClass == Boolean.TYPE ? Boolean.class : paramClass); + } + + return new ParamDescriptor( + param.name(), + defaultExpr, + param.named(), + param.positional(), + allowedClasses, + disabledByFlag); + } + + /** @see Param#name() */ + String getName() { + return name; + } + + /** Returns a description of allowed argument types suitable for an error message. */ + String getTypeErrorMessage() { + // Result has one of these forms: + // "a" + // "a or b" + // "a, b, or c" + if (allowedClasses == null) { + return Starlark.classType(Object.class); + } + StringBuilder buf = new StringBuilder(); + // TODO(b/200065655#comment3): Remove when we have an official way for package defaults. + ImmutableList> allowedClassesFiltered = + allowedClasses.stream() + .filter(x -> !Starlark.classType(x).equals("NativeComputedDefault")) + .collect(ImmutableList.toImmutableList()); + for (int i = 0, n = allowedClassesFiltered.size(); i < n; i++) { + if (i > 0) { + buf.append(n == 2 ? " or " : i < n - 1 ? ", " : ", or "); + } + buf.append(Starlark.classType(allowedClassesFiltered.get(i))); + } + return buf.toString(); + } + + @Nullable + List> getAllowedClasses() { + return allowedClasses; + } + + /** @see Param#positional() */ + boolean isPositional() { + return positional; + } + + /** @see Param#named() */ + boolean isNamed() { + return named; + } + + /** Returns the effective default value of this parameter, or null if mandatory. */ + @Nullable + Object getDefaultValue() { + return defaultValue; + } + + /** Returns the flag responsible for disabling this parameter, or null if it is enabled. */ + @Nullable + String disabledByFlag() { + return disabledByFlag; + } + + // A memoization of evalDefault, keyed by expression. + // This cache is manually maintained (instead of using LoadingCache), + // as default values may sometimes be recursively requested. + private static final ConcurrentHashMap defaultValueCache = + new ConcurrentHashMap<>(); + + // Evaluates the default value expression for a parameter. + private static Object evalDefault(String name, String expr) { + // Values required by defaults of functions in UNIVERSE must + // be handled without depending on the evaluator, or even + // on defaultValueCache, because JVM global variable initialization + // is such a mess. (Specifically, it's completely dynamic, + // so if two or more variables are mutually dependent, like + // defaultValueCache and UNIVERSE would be, you have to write + // code that works in all possible dynamic initialization orders.) + // Better not to go there. + if (expr.equals("None")) { + return Starlark.NONE; + } else if (expr.equals("True")) { + return true; + } else if (expr.equals("False")) { + return false; + } else if (expr.equals("unbound")) { + return Starlark.UNBOUND; + } else if (expr.equals("0")) { + return StarlarkInt.of(0); + } else if (expr.equals("1")) { + return StarlarkInt.of(1); + } else if (expr.equals("[]")) { + return StarlarkList.empty(); + } else if (expr.equals("()")) { + return Tuple.empty(); + } else if (expr.equals("\" \"")) { + return " "; + } + + Object x = defaultValueCache.get(expr); + if (x != null) { + return x; + } + + // We can't evaluate Starlark code until UNIVERSE is bootstrapped. + if (Starlark.UNIVERSE == null) { + throw new IllegalStateException("no bootstrap value for " + name + "=" + expr); + } + + Module module = Module.create(); + try (Mutability mu = Mutability.create("Builtin param default init")) { + // Note that this Starlark thread ignores command line flags. + // TODO: b/326588519 - The known default parameters are all simple values. If that changes, a + // non-transient symbol generator would be needed here. + StarlarkThread thread = StarlarkThread.createTransient(mu, StarlarkSemantics.DEFAULT); + + // Disable polling of the java.lang.Thread.interrupt flag during + // Starlark evaluation. Assuming the expression does not call a + // built-in that throws InterruptedException, this allows us to + // assert that InterruptedException "can't happen". + // + // Bazel Java threads are routinely interrupted during Starlark execution, + // and the Starlark interpreter may be in a call to LoadingCache (in CallUtils). + // LoadingCache computes the cache entry in the same thread that first + // requested the entry, propagating undesirable thread state (which Einstein + // called "spooky action at a distance") from an arbitrary application thread + // to here, which is logically one-time initialization code. + // + // A simpler non-solution would be to use a "clean" pool thread + // to compute each cache entry; we could safely assume such a thread + // is never interrupted. However, this runs afoul of JVM class initialization: + // the initialization of Starlark.UNIVERSE depends on Starlark.UNBOUND + // because of the reference above. That's fine if they are initialized by + // the same thread, as JVM class initialization locks are reentrant, + // but the reference deadlocks if made from another thread. + // See https://docs.oracle.com/javase/specs/jls/se12/html/jls-12.html#jls-12.4 + thread.ignoreThreadInterrupts(); + + x = Starlark.eval(ParserInput.fromLines(expr), FileOptions.DEFAULT, module, thread); + } catch (InterruptedException ex) { + throw new IllegalStateException(ex); // can't happen + } catch (SyntaxError.Exception | EvalException ex) { + throw new IllegalArgumentException( + String.format( + "failed to evaluate default value '%s' of parameter '%s': %s", + expr, name, ex.getMessage()), + ex); + } + defaultValueCache.put(expr, x); + return x; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Printer.java b/third_party/bazel/main/java/net/starlark/java/eval/Printer.java new file mode 100644 index 000000000..8bfc4626e --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/Printer.java @@ -0,0 +1,423 @@ +// Copyright 2015 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.eval; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.util.Arrays; +import java.util.IllegalFormatException; +import java.util.List; +import java.util.Map; +import java.util.MissingFormatWidthException; + +/** + * A printer of Starlark values. + * + *

Subclasses may override methods such as {@link #repr} and {@link #printList} to alter the + * formatting behavior. + */ +// TODO(adonovan): disallow printing of objects that are not Starlark values. +public class Printer { + + private final StringBuilder buffer; + + // Stack of values in the middle of being printed. + // Each renders as "..." if recursively encountered, + // indicating a cycle. + private Object[] stack; + private int depth; + + /** Creates a printer that writes to the given buffer. */ + public Printer(StringBuilder buffer) { + this.buffer = buffer; + } + + /** Creates a printer that uses a fresh buffer. */ + public Printer() { + this(new StringBuilder()); + } + + /** Appends a char to the printer's buffer */ + @CanIgnoreReturnValue + public final Printer append(char c) { + buffer.append(c); + return this; + } + + /** Appends a char sequence to the printer's buffer */ + @CanIgnoreReturnValue + public final Printer append(CharSequence s) { + buffer.append(s); + return this; + } + + /** Appends a char subsequence to the printer's buffer */ + @CanIgnoreReturnValue + public final Printer append(CharSequence s, int start, int end) { + buffer.append(s, start, end); + return this; + } + + /** Appends an integer to the printer's buffer */ + @CanIgnoreReturnValue + public final Printer append(int i) { + buffer.append(i); + return this; + } + + /** Appends a long integer to the printer's buffer */ + @CanIgnoreReturnValue + public final Printer append(long l) { + buffer.append(l); + return this; + } + + /** + * Appends a list to the printer's buffer. List elements are rendered with {@code repr}. + * + *

May be overridden by subclasses. + * + * @param list the list of objects to repr (each as with repr) + * @param before a string to print before the list items, e.g. an opening bracket + * @param separator a separator to print between items + * @param after a string to print after the list items, e.g. a closing bracket + */ + public Printer printList(Iterable list, String before, String separator, String after) { + this.append(before); + String sep = ""; + for (Object elem : list) { + this.append(sep); + sep = separator; + this.repr(elem); + } + return this.append(after); + } + + @Override + public final String toString() { + return buffer.toString(); + } + + /** + * Appends the {@code StarlarkValue.debugPrint} representation of a value (as used by the Starlark + * {@code print} statement) to the printer's buffer. + * + *

Implementations of StarlarkValue may define their own behavior of {@code debugPrint}. + */ + public Printer debugPrint(Object o, StarlarkThread thread) { + if (o instanceof StarlarkValue) { + ((StarlarkValue) o).debugPrint(this, thread); + return this; + } + + return this.str(o, thread.getSemantics()); + } + + /** + * Appends the {@code StarlarkValue.str} representation of a value to the printer's buffer. Unlike + * {@code repr(x)}, it does not quote strings at top level, though strings and other values + * appearing as elements of other structures are quoted as if by {@code repr}. + * + *

Implementations of StarlarkValue may define their own behavior of {@code str}. + */ + public Printer str(Object o, StarlarkSemantics semantics) { + if (o instanceof String) { + return this.append((String) o); + + } else if (o instanceof StarlarkValue) { + ((StarlarkValue) o).str(this, semantics); + return this; + + } else { + return this.repr(o); + } + } + + /** + * Appends the {@code StarlarkValue.repr} (quoted) representation of a value to the printer's + * buffer. The quoted form is often a Starlark expression that evaluates to the value. + * + *

Implementations of StarlarkValue may define their own behavior of {@code repr}. + * + *

Cyclic values are rendered as {@code ...} if they are recursively encountered by the same + * printer. (Implementations of {@link StarlarkValue#repr} should avoid calling {@code + * Starlark#repr}, as it creates another printer, which hide cycles from the cycle detector.) + * + *

In addition to Starlark values, {@code repr} also prints instances of classes Map, List, + * Map.Entry, or Class. All other values are formatted using their {@code toString} method. + * TODO(adonovan): disallow that. + */ + public Printer repr(Object o) { + // atomic values (leaves of the object graph) + if (o == null) { + // Java null is not a valid Starlark value, but sometimes printers are used on non-Starlark + // values such as Locations or Nodes. + return this.append("null"); + + } else if (o instanceof String) { + appendQuoted((String) o); + return this; + + } else if (o instanceof StarlarkInt) { + ((StarlarkInt) o).repr(this); + return this; + + } else if (o instanceof Boolean) { + this.append(((boolean) o) ? "True" : "False"); + return this; + + } else if (o instanceof Integer) { // a non-Starlark value + this.buffer.append((int) o); + return this; + + } else if (o instanceof Class) { // a non-Starlark value + this.append(Starlark.classType((Class) o)); + return this; + } + + // compound values (may form cycles in the object graph) + + if (!push(o)) { + return this.append("..."); // elided cycle + } + try { + if (o instanceof StarlarkValue) { + ((StarlarkValue) o).repr(this); + + // -- non-Starlark values -- + + } else if (o instanceof Map) { + Map dict = (Map) o; + this.printList(dict.entrySet(), "{", ", ", "}"); + + } else if (o instanceof List) { + this.printList((List) o, "[", ", ", "]"); + + } else if (o instanceof Map.Entry) { + Map.Entry entry = (Map.Entry) o; + this.repr(entry.getKey()); + this.append(": "); + this.repr(entry.getValue()); + + } else { + // All other non-Starlark Java values (e.g. Node, Location). + // Starlark code cannot access values of o that would reach here, + // and native code is already trusted to be deterministic. + this.append(o.toString()); + } + } finally { + pop(); + } + + return this; + } + + private Printer appendQuoted(String s) { + this.append('"'); + int len = s.length(); + for (int i = 0; i < len; i++) { + char c = s.charAt(i); + escapeCharacter(c); + } + return this.append('"'); + } + + private Printer backslashChar(char c) { + return this.append('\\').append(c); + } + + private Printer escapeCharacter(char c) { + if (c == '"') { + return backslashChar(c); + } + switch (c) { + case '\\': + return backslashChar('\\'); + case '\r': + return backslashChar('r'); + case '\n': + return backslashChar('n'); + case '\t': + return backslashChar('t'); + default: + if (c < 32) { + // TODO(bazel-team): support \x escapes + return this.append(String.format("\\x%02x", (int) c)); + } + return this.append(c); // no need to support UTF-8 + } + } + + // Reports whether x is already present on the visitation stack, pushing it if not. + private boolean push(Object x) { + // cyclic? + for (int i = 0; i < depth; i++) { + if (x == stack[i]) { + return false; + } + } + + if (stack == null) { + this.stack = new Object[4]; + } else if (depth == stack.length) { + this.stack = Arrays.copyOf(stack, 2 * stack.length); + } + this.stack[depth++] = x; + return true; + } + + private void pop() { + this.stack[--depth] = null; + } + + /** + * Appends a string, formatted as if by Starlark's {@code str % tuple} operator, to the printer's + * buffer. + * + *

Supported conversions: + * + *

    + *
  • {@code %s} (convert as if by {@code str()}) + *
  • {@code %r} (convert as if by {@code repr()}) + *
  • {@code %d} (convert an integer to its decimal representation) + *
+ * + * To encode a literal percent character, escape it as {@code %%}. It is an error to have a + * non-escaped {@code %} at the end of the string or followed by any character not listed above. + * + * @param format the format string + * @param arguments an array containing arguments to substitute into the format operators in order + * @throws IllegalFormatException if the format string is invalid or the arguments do not match it + */ + public static void format( + Printer printer, StarlarkSemantics semantics, String format, Object... arguments) { + formatWithList(printer, semantics, format, Arrays.asList(arguments)); + } + + /** Same as {@link #format}, but with a list instead of variadic args. */ + @SuppressWarnings("FormatString") // see b/178189609 + public static void formatWithList( + Printer printer, StarlarkSemantics semantics, String pattern, List arguments) { + // N.B. MissingFormatWidthException is the only kind of IllegalFormatException + // whose constructor can take and display arbitrary error message, hence its use below. + // TODO(adonovan): this suggests we're using the wrong exception. Throw IAE? + + int length = pattern.length(); + int argLength = arguments.size(); + int i = 0; // index of next character in pattern + int a = 0; // index of next argument in arguments + + while (i < length) { + int p = pattern.indexOf('%', i); + if (p == -1) { + printer.append(pattern, i, length); + break; + } + if (p > i) { + printer.append(pattern, i, p); + } + if (p == length - 1) { + throw new MissingFormatWidthException( + "incomplete format pattern ends with %: " + Starlark.repr(pattern)); + } + char conv = pattern.charAt(p + 1); + i = p + 2; + + // %%: literal % + if (conv == '%') { + printer.append('%'); + continue; + } + + // get argument + if (a >= argLength) { + throw new MissingFormatWidthException( + "not enough arguments for format pattern " + + Starlark.repr(pattern) + + ": " + + Starlark.repr(Tuple.copyOf(arguments))); + } + Object arg = arguments.get(a++); + + switch (conv) { + case 'd': + case 'o': + case 'x': + case 'X': + { + Number n; + if (arg instanceof StarlarkInt) { + n = ((StarlarkInt) arg).toNumber(); + } else if (arg instanceof Integer) { + n = (Number) arg; + } else if (arg instanceof StarlarkFloat) { + double d = ((StarlarkFloat) arg).toDouble(); + try { + n = StarlarkInt.ofFiniteDouble(d).toNumber(); + } catch (IllegalArgumentException unused) { + throw new MissingFormatWidthException("got " + arg + ", want a finite number"); + } + } else { + throw new MissingFormatWidthException( + String.format( + "got %s for '%%%c' format, want int or float", Starlark.type(arg), conv)); + } + printer.append( + String.format( + conv == 'd' ? "%d" : conv == 'o' ? "%o" : conv == 'x' ? "%x" : "%X", n)); + continue; + } + + case 'e': + case 'f': + case 'g': + case 'E': + case 'F': + case 'G': + double v; + if (arg instanceof Integer) { + v = (double) (Integer) arg; + } else if (arg instanceof StarlarkInt) { + v = ((StarlarkInt) arg).toDouble(); + } else if (arg instanceof StarlarkFloat) { + v = ((StarlarkFloat) arg).toDouble(); + } else { + throw new MissingFormatWidthException( + String.format( + "got %s for '%%%c' format, want int or float", Starlark.type(arg), conv)); + } + printer.append(StarlarkFloat.format(v, conv)); + continue; + + case 'r': + printer.repr(arg); + continue; + + case 's': + printer.str(arg, semantics); + continue; + + default: + // The call to Starlark.repr doesn't cause an infinite recursion + // because it's only used to format a string properly. + throw new MissingFormatWidthException( + String.format( + "unsupported format character \"%s\" at index %s in %s", + String.valueOf(conv), p + 1, Starlark.repr(pattern))); + } + } + if (a < argLength) { + throw new MissingFormatWidthException("not all arguments converted during string formatting"); + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/RangeList.java b/third_party/bazel/main/java/net/starlark/java/eval/RangeList.java new file mode 100644 index 000000000..bd719bcec --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/RangeList.java @@ -0,0 +1,214 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.base.Preconditions; +import com.google.common.collect.UnmodifiableIterator; +import java.util.AbstractList; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Objects; +import javax.annotation.concurrent.Immutable; +import net.starlark.java.annot.StarlarkBuiltin; + +/** + * A sequence returned by the {@code range} function invocation. + * + *

Instead of eagerly allocating an array with all elements of the sequence, this class uses + * simple math to compute a value at each index. This is particularly useful when range is huge or + * only a few elements from it are used. + * + *

The start, stop, step, and size of the range must all fit within 32-bit signed integers. + * + *

Eventually {@code range} function should produce an instance of the {@code range} type as is + * the case in Python 3, but for now to preserve backwards compatibility with Python 2, {@code list} + * is returned. + */ +@StarlarkBuiltin( + name = "range", + category = "core", + doc = + "A language built-in type to support ranges. Example of range literal:
" + + "

x = range(1, 10, 3)
" + + "Accessing elements is possible using indexing (starts from 0):
" + + "
e = x[1]   # e == 2
" + + "Ranges do not support the + operator for concatenation." + + "Similar to strings, ranges support slice operations:" + + "
range(10)[1:3]   # range(1, 3)\n"
+            + "range(10)[::2]  # range(0, 10, 2)\n"
+            + "range(10)[3:0:-1]  # range(3, 0, -1)
" + + "Ranges are immutable, as in Python 3.") +@Immutable +final class RangeList extends AbstractList implements Sequence { + + private final int start; + private final int stop; + private final int step; + private final int size; // (derived) + + RangeList(int start, int stop, int step) throws EvalException { + Preconditions.checkArgument(step != 0); + + this.start = start; + this.stop = stop; + this.step = step; + + // compute size. + // Python version: + // https://github.com/python/cpython/blob/09bb918a61031377d720f1a0fa1fe53c962791b6/Objects/rangeobject.c#L144 + int low; // [low,high) is a half-open interval + int high; + long absStep; + if (step > 0) { + low = start; + high = stop; + absStep = step; + } else { + low = stop; + high = start; + absStep = -(long) step; + } + if (low >= high) { + this.size = 0; + } else { + long diff = (long) high - low - 1; + long size = diff / absStep + 1; + if ((int) size != size) { + throw Starlark.errorf("len(%s) exceeds signed 32-bit range", Starlark.repr(this)); + } + this.size = (int) size; + } + } + + @Override + public boolean contains(Object x) { + if (!(x instanceof StarlarkInt)) { + return false; + } + try { + int i = ((StarlarkInt) x).toIntUnchecked(); + + // constant-time implementation + if (step > 0) { + return start <= i && i < stop && (i - start) % step == 0; + } else { + return stop < i && i <= start && (i - start) % step == 0; + } + } catch (IllegalArgumentException ex) { + return false; // x is not a signed 32-bit int + } + } + + @Override + public StarlarkInt get(int index) { + if (index < 0 || index >= size()) { + throw new ArrayIndexOutOfBoundsException(index + ":" + this); + } + return StarlarkInt.of(at(index)); + } + + @Override + public int size() { + return size; + } + + @Override + public int hashCode() { + if (size == 0) { + return 234982346; + } else if (size == 1) { + return Integer.hashCode(start); + } else { + return Objects.hash(start, size, step); + } + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof RangeList)) { + return false; + } + RangeList that = (RangeList) other; + + // Two RangeLists compare equal if they denote the same sequence. + if (this.size != that.size) { + return false; // sequences differ in length + } + if (this.size == 0) { + return true; // both sequences are empty + } + if (this.start != that.start) { + return false; // first element differs + } + return this.size == 1 || this.step == that.step; + } + + @Override + public Iterator iterator() { + return new UnmodifiableIterator() { + long cursor = start; // returned by next() if hasNext() is true + + @Override + public boolean hasNext() { + return (step > 0) ? (cursor < stop) : (cursor > stop); + } + + @Override + public StarlarkInt next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + // If cursor is valid, it's guaranteed to be in [start, stop) range, thus a 32-bit value. + int current = (int) cursor; + cursor += step; + return StarlarkInt.of(current); + } + }; + } + + @Override + public Sequence getSlice(Mutability mu, int start, int stop, int step) + throws EvalException { + long sliceStep = (long) step * (long) this.step; + if (sliceStep != (int) sliceStep) { + // It is not an error to take a slice of a RangeList such that the slice step * list step + // doesn't fit in a 32-bit int; the result ought to be a RangeList containing only one + // element (the start). Since difference between 2 successive elements of a RangeList must be + // a 32-bit int, clamping the step to Integer.MAX_VALUE or MIN_VALUE and moving stop to start + // +/- 1 gives us the 1-element RangeList we need. + sliceStep = sliceStep > 0 ? Integer.MAX_VALUE : Integer.MIN_VALUE; // note sliceStep != 0 + if (stop > start) { + stop = start + 1; + } else if (stop < start) { + stop = start - 1; + } + } + return new RangeList(at(start), at(stop), (int) sliceStep); + } + + // Like get, but without bounds check or Integer allocation. + int at(int i) { + return start + step * i; + } + + @Override + public void repr(Printer printer) { + if (step == 1) { + printer.append(String.format("range(%d, %d)", start, stop)); + } else { + printer.append(String.format("range(%d, %d, %d)", start, stop, step)); + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/RegularImmutableStarlarkList.java b/third_party/bazel/main/java/net/starlark/java/eval/RegularImmutableStarlarkList.java new file mode 100644 index 000000000..07b7df983 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/RegularImmutableStarlarkList.java @@ -0,0 +1,42 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.base.Preconditions; + +/** An immutable implementation of a {@code StarlarkList}. */ +final class RegularImmutableStarlarkList extends ImmutableStarlarkList { + + /** + * A shared instance for the empty list with immutable mutability. + * + *

Other immutable empty list objects can exist, e.g. lists that were once mutable but whose + * environments were then frozen. This instance is for empty lists that were always frozen from + * the beginning. + */ + static final StarlarkList EMPTY = new RegularImmutableStarlarkList<>(EMPTY_ARRAY); + + private final Object[] elems; + + RegularImmutableStarlarkList(Object[] elems) { + Preconditions.checkArgument(elems.getClass() == Object[].class); + this.elems = elems; + } + + @Override + Object[] elems() { + return elems; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/RegularTuple.java b/third_party/bazel/main/java/net/starlark/java/eval/RegularTuple.java new file mode 100644 index 000000000..f65d9df47 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/RegularTuple.java @@ -0,0 +1,159 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import java.util.Arrays; + +/** + * An implementation of non-singleton (empty or more than 1 element) Tuple using an {@code Object} + * array. + */ +final class RegularTuple extends Tuple { + + // The shared (sole) empty tuple. + static final Tuple EMPTY = new RegularTuple(new Object[] {}); + + final Object[] elems; + + RegularTuple(Object[] elems) { + Preconditions.checkArgument(elems.length != 1); + this.elems = elems; + } + + @Override + public boolean isImmutable() { + for (Object x : elems) { + if (!Starlark.isImmutable(x)) { + return false; + } + } + return true; + } + + @Override + public void checkHashable() throws EvalException { + for (Object x : elems) { + Starlark.checkHashable(x); + } + } + + @Override + public int hashCode() { + return 9857 + 8167 * Arrays.hashCode(elems); + } + + @Override + public Object get(int i) { + return elems[i]; + } + + @Override + public int size() { + return elems.length; + } + + @Override + public boolean contains(Object o) { + // Tuple contains only valid Starlark objects (which are non-null) + if (o == null) { + return false; + } + for (Object elem : elems) { + if (o.equals(elem)) { + return true; + } + } + return false; + } + + @Override + public Tuple subList(int from, int to) { + Preconditions.checkPositionIndexes(from, to, elems.length); + return wrap(Arrays.copyOfRange(elems, from, to)); + } + + /** Returns a new array of class Object[] containing the tuple elements. */ + @Override + public Object[] toArray() { + return elems.length != 0 ? Arrays.copyOf(elems, elems.length, Object[].class) : elems; + } + + @SuppressWarnings("unchecked") + @Override + public T[] toArray(T[] a) { + if (a.length < elems.length) { + return (T[]) Arrays.copyOf(elems, elems.length, a.getClass()); + } else { + System.arraycopy(elems, 0, a, 0, elems.length); + Arrays.fill(a, elems.length, a.length, null); + return a; + } + } + + @Override + public void repr(Printer printer) { + // Remark: singletons, are represented as {@code '(x,)'}; whereas this implementation would + // return + // {@code '(x)'} + printer.append('('); + String sep = ""; + for (Object elem : elems) { + printer.append(sep); + sep = ", "; + printer.repr(elem); + } + printer.append(')'); + } + + @Override + public ImmutableList getImmutableList() { + // Share the array with this (immutable) Tuple. + return wrapImmutable(elems); + } + + @Override + public Tuple getSlice(Mutability mu, int start, int stop, int step) throws EvalException { + RangeList indices = new RangeList(start, stop, step); + int n = indices.size(); + if (step == 1) { // common case + return subList(indices.at(0), indices.at(n)); + } + Object[] res = new Object[n]; + for (int i = 0; i < n; ++i) { + res[i] = elems[indices.at(i)]; + } + return wrap(res); + } + + @Override + Tuple repeat(StarlarkInt n) throws EvalException { + if (n.signum() <= 0 || isEmpty()) { + return empty(); + } + + int ni = n.toInt("repeat"); + long sz = (long) ni * elems.length; + if (sz > StarlarkList.MAX_ALLOC) { + throw Starlark.errorf("excessive repeat (%d * %d elements)", elems.length, ni); + } + Object[] res = new Object[(int) sz]; + for (int i = 0; i < ni; i++) { + System.arraycopy(elems, 0, res, i * elems.length, elems.length); + } + return wrap(res); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Sequence.java b/third_party/bazel/main/java/net/starlark/java/eval/Sequence.java new file mode 100644 index 000000000..544f9d35c --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/Sequence.java @@ -0,0 +1,157 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import static java.lang.Math.min; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import java.util.List; +import java.util.RandomAccess; + +/** + * A Sequence is a finite iterable sequence of Starlark values, such as a list or tuple. + * + *

Sequences implement the read-only operations of the {@link List} interface, but not its update + * operations, similar to {@code ImmutableList}. The specification of {@code List} governs how such + * methods behave and in particular how they report errors. Subclasses of sequence may define ad-hoc + * mutator methods, such as {@link StarlarkList#extend}, exposed to Starlark, or Java, or both. + * + *

In principle, subclasses of Sequence could also define the standard update operations of List, + * but there appears to be little demand, and doing so carries some risk of obscuring unintended + * mutations to Starlark values that would currently cause the program to crash. + */ +public interface Sequence + extends StarlarkValue, List, RandomAccess, StarlarkIndexable, StarlarkIterable { + + @Override + default boolean truth() { + return !isEmpty(); + } + + /** Returns an ImmutableList object with the current underlying contents of this Sequence. */ + default ImmutableList getImmutableList() { + return ImmutableList.copyOf(this); + } + + /** Retrieves an entry from a Sequence. */ + @Override + default E getIndex(StarlarkSemantics semantics, Object key) throws EvalException { + int index = Starlark.toInt(key, "sequence index"); + return get(EvalUtils.getSequenceIndex(index, size())); + } + + @Override + default boolean containsKey(StarlarkSemantics semantics, Object key) throws EvalException { + return contains(key); + } + + /** + * Compares two sequences of values. Sequences compare equal if corresponding elements compare + * equal using {@code x[i] == y[i]}. Otherwise, the result is the ordered comparison of the first + * element for which {@code x[i] != y[i]}. If one list is a prefix of another, the result is the + * comparison of the list's sizes. + * + * @throws ClassCastException if any comparison failed. + */ + static int compare(List x, List y) { + for (int i = 0; i < min(x.size(), y.size()); i++) { + Object xelem = x.get(i); + Object yelem = y.get(i); + + // First test for equality. This avoids an unnecessary + // ordered comparison, which may be unsupported despite + // the values being equal. Also, it is potentially more + // expensive. For example, list==list need not look at + // the elements if the lengths are unequal. + if (xelem == yelem || xelem.equals(yelem)) { + continue; + } + + // The ordered comparison of unequal elements should + // always be nonzero unless compareTo is inconsistent. + int cmp = Starlark.compareUnchecked(xelem, yelem); + if (cmp == 0) { + throw new IllegalStateException( + String.format( + "x.equals(y) yet x.compareTo(y)==%d (x: %s, y: %s)", + cmp, Starlark.type(xelem), Starlark.type(yelem))); + } + return cmp; + } + return Integer.compare(x.size(), y.size()); + } + + /** + * Compares two sequences of value for equality. Sequences compare equal if they are the same size + * and corresponding elements compare equal. + */ + static boolean sameElems(List x, List y) { + if (x == y) { + return true; + } + if (x.size() != y.size()) { + return false; + } + for (int i = 0; i < x.size(); i++) { + Object xelem = x.get(i); + Object yelem = y.get(i); + + if (xelem != yelem && !xelem.equals(yelem)) { + return false; + } + } + return true; + } + + /** + * Returns the slice of this sequence, {@code this[start:stop:step]}.
+ * For positive strides ({@code step > 0}), {@code 0 <= start <= stop <= size()}.
+ * For negative strides ({@code step < 0}), {@code -1 <= stop <= start < size()}.
+ * The caller must ensure that the start and stop indices are valid and that step is non-zero. + */ + Sequence getSlice(Mutability mu, int start, int stop, int step) throws EvalException; + + /** + * Casts a non-null Starlark value {@code x} to a {@code Sequence}, after checking that each + * element is an instance of {@code elemType}. On error, it throws an EvalException whose message + * includes {@code what}, ideally a string literal, as a description of the role of {@code x}. + */ + public static Sequence cast(Object x, Class elemType, String what) + throws EvalException { + Preconditions.checkNotNull(x); + if (!(x instanceof Sequence)) { + throw Starlark.errorf("for %s, got %s, want sequence", what, Starlark.type(x)); + } + int i = 0; + for (Object elem : (Sequence) x) { + if (!elemType.isAssignableFrom(elem.getClass())) { + throw Starlark.errorf( + "at index %d of %s, got element of type %s, want %s", + i, what, Starlark.type(elem), Starlark.classType(elemType)); + } + i++; + } + @SuppressWarnings("unchecked") // safe + Sequence result = (Sequence) x; + return result; + } + + /** Like {@link #cast}, but if x is None, returns an immutable empty list. */ + public static Sequence noneableCast(Object x, Class type, String what) + throws EvalException { + return x == Starlark.NONE ? StarlarkList.empty() : cast(x, type, what); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/SingletonTuple.java b/third_party/bazel/main/java/net/starlark/java/eval/SingletonTuple.java new file mode 100644 index 000000000..f5d5e6893 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/SingletonTuple.java @@ -0,0 +1,119 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import java.util.Arrays; + +/** A specific implementation of a Tuple that has exactly 1 element. */ +final class SingletonTuple extends Tuple { + final Object elem; + + SingletonTuple(Object elem) { + this.elem = elem; + } + + @Override + public boolean isImmutable() { + return Starlark.isImmutable(elem); + } + + @Override + public void checkHashable() throws EvalException { + Starlark.checkHashable(elem); + } + + @Override + public int hashCode() { + // This produces the same results as RegularTuple.hashCode(), + return 9857 + 8167 * (31 + elem.hashCode()); + } + + @Override + public Object get(int i) { + Preconditions.checkElementIndex(i, 1); + return elem; + } + + @Override + public int size() { + return 1; + } + + @Override + public boolean contains(Object o) { + // Tuple contains only valid Starlark objects (which are non-null) + if (o == null) { + return false; + } + return o.equals(elem); + } + + @Override + public Tuple subList(int from, int to) { + Preconditions.checkPositionIndexes(from, to, 1); + return from <= 0 && to >= 1 ? this : empty(); + } + + /** Returns a new array of class Object[] containing the tuple elements. */ + @Override + public Object[] toArray() { + return new Object[] {elem}; + } + + @SuppressWarnings("unchecked") + @Override + public T[] toArray(T[] a) { + if (a.length < 1) { + return (T[]) new Object[] {elem}; + } else { + a[0] = (T) elem; + Arrays.fill(a, 1, a.length, null); + return a; + } + } + + @Override + public void repr(Printer printer) { + printer.append('(').repr(elem).append(",)"); + } + + @Override + public ImmutableList getImmutableList() { + return ImmutableList.of(elem); + } + + @Override + public Tuple getSlice(Mutability mu, int start, int stop, int step) throws EvalException { + RangeList indices = new RangeList(start, stop, step); + return indices.isEmpty() ? Tuple.empty() : this; + } + + @Override + Tuple repeat(StarlarkInt n) throws EvalException { + if (n.signum() <= 0) { + return empty(); + } + + int ni = n.toInt("repeat"); + if (ni > StarlarkList.MAX_ALLOC) { + throw Starlark.errorf("excessive repeat (%d * %d elements)", 1, ni); + } + Object[] res = new Object[ni]; + Arrays.fill(res, 0, ni, elem); + return wrap(res); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Starlark.java b/third_party/bazel/main/java/net/starlark/java/eval/Starlark.java new file mode 100644 index 000000000..e36e7da6d --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/Starlark.java @@ -0,0 +1,1179 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.eval; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.Math.min; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; +import com.google.errorprone.annotations.CheckReturnValue; +import com.google.errorprone.annotations.FormatMethod; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.reflect.Method; +import java.math.BigInteger; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; +import net.starlark.java.annot.StarlarkAnnotations; +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; +import net.starlark.java.spelling.SpellChecker; +import net.starlark.java.syntax.Expression; +import net.starlark.java.syntax.FileOptions; +import net.starlark.java.syntax.ParserInput; +import net.starlark.java.syntax.Program; +import net.starlark.java.syntax.Resolver; +import net.starlark.java.syntax.StarlarkFile; +import net.starlark.java.syntax.SyntaxError; + +/** + * The Starlark class defines the most important entry points, constants, and functions needed by + * all clients of the Starlark interpreter. + */ +public final class Starlark { + + private Starlark() {} // uninstantiable + + /** The Starlark None value. */ + public static final NoneType NONE = NoneType.NONE; + + /** + * A sentinel value passed to optional parameters of StarlarkMethod-annotated methods to indicate + * that no argument value was supplied. + */ + public static final Object UNBOUND = new UnboundMarker(); + + /** A type representing no argument passed to {@code StarlarkMethod}s */ + @Immutable + public static final class UnboundMarker implements StarlarkValue { + private UnboundMarker() {} + + @Override + public String toString() { + return ""; + } + + @Override + public boolean isImmutable() { + return true; + } + + @Override + public void repr(Printer printer) { + printer.append(""); + } + } + + /** + * The universal bindings predeclared in every Starlark file, such as None, True, len, and range. + */ + public static final ImmutableMap UNIVERSE = makeUniverse(); + + /** + * An {@code IllegalArgumentException} subclass for when a non-Starlark object is encountered in a + * context where a Starlark value ({@code String}, {@code Boolean}, or {@code StarlarkValue}) was + * expected. + */ + public static final class InvalidStarlarkValueException extends IllegalArgumentException { + @Nullable private final Class invalidClass; + + public Class getInvalidClass() { + return invalidClass; + } + + private InvalidStarlarkValueException(@Nullable Class invalidClass) { + super("invalid Starlark value: " + (invalidClass == null ? "null" : invalidClass)); + this.invalidClass = invalidClass; + } + } + + private static ImmutableMap makeUniverse() { + ImmutableMap.Builder env = ImmutableMap.builder(); + env // + .put("False", false) + .put("True", true) + .put("None", NONE); + addMethods(env, new MethodLibrary()); + return env.build(); + } + + /** + * Reports whether the argument is a legal Starlark value: a string, boolean, or StarlarkValue. + */ + public static boolean valid(Object x) { + return x instanceof String || x instanceof Boolean || x instanceof StarlarkValue; + } + + /** + * Returns {@code x} if it is a {@link #valid} Starlark value, otherwise throws + * InvalidStarlarkValueException. + */ + public static T checkValid(T x) { + if (!valid(x)) { + throw new InvalidStarlarkValueException(x == null ? null : x.getClass()); + } + return x; + } + + /** Reports whether {@code x} is Java null or Starlark None. */ + public static boolean isNullOrNone(Object x) { + return x == null || x == NONE; + } + + /** Reports whether a Starlark value is assumed to be deeply immutable. */ + // TODO(adonovan): eliminate the concept of querying for immutability. It is currently used for + // only one purpose, the precondition for adding an element to a Depset, but Depsets should check + // hashability, like Dicts. (Similarly, querying for hashability should go: just attempt to hash a + // value, and be prepared for it to fail.) In practice, a value may be immutable, either + // inherently (e.g. string) or because it has become frozen, but we don't need to query for it. + // Just attempt a mutation and be prepared for it to fail. + // It is inefficient and potentially inconsistent to ask before doing. + // + // The main obstacle is that although depsets disallow (say) lists as keys even when frozen, + // they permit a tuple of lists, or a struct containing lists, and many users exploit this. + public static boolean isImmutable(Object x) { + // NB: This is used as the basis for accepting objects in Depsets, + // as well as for accepting objects as keys for Starlark dicts. + + if (x instanceof String || x instanceof Boolean) { + return true; + } else if (x instanceof StarlarkValue) { + return ((StarlarkValue) x).isImmutable(); + } else { + throw new InvalidStarlarkValueException(x.getClass()); + } + } + + /** + * Returns normally if the Starlark value is hashable and thus suitable as a dict key. + * + * @throws EvalException otherwise. + */ + public static void checkHashable(Object x) throws EvalException { + if (x instanceof String) { + // Strings are the most common dict keys. Check them first, since `instanceof StarlarkValue` + // (an interface) is slower than `instanceof String` (a final class). + } else if (x instanceof StarlarkValue) { + ((StarlarkValue) x).checkHashable(); + } else { + // Throw if the type is bad. Otherwise it's a Boolean, which is hashable. + Starlark.checkValid(x); + } + } + + /** + * Converts a Java value {@code x} to a Starlark one, if x is not already a valid Starlark value. + * An Integer, Long, or BigInteger is converted to a Starlark int, a double is converted to a + * Starlark float, a Java List or Map is converted to a Starlark list or dict, respectively, and + * null becomes {@link #NONE}. Any other non-Starlark value causes the function to throw + * InvalidStarlarkValueException. + * + *

Elements of Lists and Maps must be valid Starlark values; they are not recursively + * converted. (This avoids excessive unintended deep copying.) + * + *

This function is applied to the results of StarlarkMethod-annotated Java methods. + */ + public static Object fromJava(Object x, @Nullable Mutability mutability) { + if (x == null) { + return NONE; + } else if (valid(x)) { + return x; + } else if (x instanceof Number) { + if (x instanceof Integer) { + return StarlarkInt.of((Integer) x); + } else if (x instanceof Long) { + return StarlarkInt.of((Long) x); + } else if (x instanceof BigInteger) { + return StarlarkInt.of((BigInteger) x); + } else if (x instanceof Double) { + return StarlarkFloat.of((double) x); + } + } else if (x instanceof List) { + return StarlarkList.copyOf(mutability, (List) x); + } else if (x instanceof Map) { + return Dict.copyOf(mutability, (Map) x); + } + throw new InvalidStarlarkValueException(x.getClass()); + } + + /** + * Converts a Starlark method's bound, non-None parameter value to a Java Optional wrapping that + * value, and an unbound or None value to an empty Optional. + * + *

This is typically used in {@link StarlarkMethod} implementations, with a parameter whose + * {@link Param#allowedTypes} is set to be {@code {T}} or {@code {NoneType, T}}. + * + * @throws ClassCastException if value is bound and non-None but is not of the expected class + */ + public static Optional toJavaOptional(Object x, Class expectedClass) { + if (x == Starlark.UNBOUND || x == Starlark.NONE) { + return Optional.empty(); + } else { + return Optional.of(expectedClass.cast(x)); + } + } + + /** + * Returns the truth value of a valid Starlark value, as if by the Starlark expression {@code + * bool(x)}. + */ + public static boolean truth(Object x) { + if (x instanceof Boolean) { + return (Boolean) x; + } else if (x instanceof StarlarkValue) { + return ((StarlarkValue) x).truth(); + } else if (x instanceof String) { + return !((String) x).isEmpty(); + } else { + throw new InvalidStarlarkValueException(x.getClass()); + } + } + + /** + * Checks whether the Freezable Starlark value is frozen or temporarily immutable due to active + * iterators. + * + * @throws EvalException if the value is not mutable. + */ + public static void checkMutable(Mutability.Freezable x) throws EvalException { + if (x.mutability().isFrozen()) { + throw errorf("trying to mutate a frozen %s value", type(x)); + } + if (x.updateIteratorCount(0)) { + throw errorf("%s value is temporarily immutable due to active for-loop iteration", type(x)); + } + } + + /** + * Returns an iterable view of {@code x} if it is an iterable Starlark value; throws EvalException + * otherwise. + * + *

Whereas the interpreter temporarily freezes the iterable value by bracketing {@code for} + * loops and comprehensions in calls to {@link Freezable#updateIteratorCount}, iteration using + * this method does not freeze the value. Callers should exercise care not to mutate the + * underlying object during iteration. + */ + public static Iterable toIterable(Object x) throws EvalException { + if (x instanceof StarlarkIterable) { + return (Iterable) x; + } + throw errorf("type '%s' is not iterable", type(x)); + } + + /** + * Returns a new array of class Object[] containing the elements of Starlark iterable value {@code + * x}. A Starlark value is iterable if it implements {@link StarlarkIterable}. + */ + public static Object[] toArray(Object x) throws EvalException { + // Specialize Sequence and Dict to avoid allocation and/or indirection. + if (x instanceof Sequence) { + // The returned array type must be exactly Object[], + // not a subclass, so calling toArray() is not enough. + return ((Sequence) x).toArray(EMPTY); + } else if (x instanceof Dict) { + return ((Dict) x).keySet().toArray(); + } else { + return Iterables.toArray(toIterable(x), Object.class); + } + } + + /** + * Returns the length of a Starlark string, sequence (such as a list or tuple), dict, or other + * iterable, as if by the Starlark expression {@code len(x)}, or -1 if the value is valid but has + * no length. + */ + public static int len(Object x) { + if (x instanceof String) { + return ((String) x).length(); + } else if (x instanceof Sequence) { + return ((Sequence) x).size(); + } else if (x instanceof Dict) { + return ((Dict) x).size(); + } else if (x instanceof StarlarkSet) { + return ((StarlarkSet) x).size(); + } else if (x instanceof StarlarkIterable) { + // Iterables.size runs in constant time if x implements Collection. + return Iterables.size((Iterable) x); + } else { + checkValid(x); + return -1; // valid but not a sequence + } + } + + /** Returns the name of the type of a value as if by the Starlark expression {@code type(x)}. */ + public static String type(Object x) { + return classType(x.getClass()); + } + + /** + * Returns the name of the type of instances of class c. + * + *

This function accepts any class, not just those of legal Starlark values, and may be used + * for reporting error messages involving arbitrary Java classes, for example at the interface + * between Starlark and Java. + */ + public static String classType(Class c) { + // Check for "direct hits" first to avoid needing to scan for annotations. + if (c.equals(String.class)) { + return "string"; + } else if (StarlarkInt.class.isAssignableFrom(c)) { + return "int"; + } else if (c.equals(Boolean.class)) { + return "bool"; + } else if (c.equals(StarlarkFloat.class)) { + return "float"; + } + + // Shortcut for the most common types. + // These cases can be handled by `getStarlarkBuiltin` + // but `getStarlarkBuiltin` is quite expensive. + if (StarlarkList.class.isAssignableFrom(c)) { + return "list"; + } else if (Tuple.class.isAssignableFrom(c)) { + return "tuple"; + } else if (c.equals(Dict.class)) { + return "dict"; + } else if (c.equals(NoneType.class)) { + return "NoneType"; + } else if (c.equals(StarlarkFunction.class)) { + return "function"; + } else if (c.equals(RangeList.class)) { + return "range"; + } else if (c.equals(UnboundMarker.class)) { + return "unbound"; + } + + // Abstract types, often used as parameter types. + // Note == not isAssignableFrom: we don't want any + // concrete types to inherit these names. + if (c == StarlarkIterable.class) { + return "iterable"; + } else if (c == Sequence.class) { + return "sequence"; + } else if (c == StarlarkCallable.class) { + return "callable"; + } else if (c == Structure.class) { + return "structure"; + } + + StarlarkBuiltin module = StarlarkAnnotations.getStarlarkBuiltin(c); + if (module != null) { + return module.name(); + } + + if (c.equals(Object.class)) { + // "unknown" is another unfortunate choice. + // Object.class does mean "unknown" when talking about the type parameter + // of a collection (List), but it also means "any" when used + // as an argument to Sequence.cast, and more generally it means "value". + return "unknown"; + + } else if (List.class.isAssignableFrom(c)) { + // Any class of java.util.List that isn't a Sequence. + return "List"; + + } else if (Map.class.isAssignableFrom(c)) { + // Any class of java.util.Map that isn't a Dict. + return "Map"; + + } else if (c.equals(Integer.class)) { + // Integer is not a legal Starlark value, but it does appear as + // the return type for many built-in functions. + return "int"; + + } else if (c == void.class) { + // Built-in void methods return None to Starlark. + return "NoneType"; + + } else if (c == boolean.class) { + // Built-in function may return boolean. + return "bool"; + + } else { + String simpleName = c.getSimpleName(); + return simpleName.isEmpty() ? c.getName() : simpleName; + } + } + + /** + * Returns the name of the type of instances of {@code c} after being converted to Starlark values + * by {@link #fromJava}, or "unknown" for {@code Object.class}, since that is used as a wildcard + * type by evaluation machinery. + * + *

Note that {@code void.class} is treated as "NoneType" since void methods will return None to + * Starlark. + * + * @throws InvalidStarlarkValueException if {@code c} is not {@code Object.class} and {@link + * #fromJava} would throw for instances of {@code c}. + */ + public static String classTypeFromJava(Class c) { + if (c.equals( + void.class) // Method.invoke on void-returning methods returns null; we treat it as None + || c.equals(String.class) + || c.equals(boolean.class) + || c.equals(Boolean.class) + || StarlarkValue.class.isAssignableFrom(c) + || c.equals(Object.class)) { + return classType(c); + } else if (c.equals(int.class) + || c.equals(Integer.class) + || c.equals(long.class) + || c.equals(Long.class) + || BigInteger.class.isAssignableFrom(c)) { + return classType(StarlarkInt.class); + } else if (c.equals(double.class) || c.equals(Double.class)) { + return classType(StarlarkFloat.class); + } else if (List.class.isAssignableFrom(c)) { + return classType(StarlarkList.class); + } else if (Map.class.isAssignableFrom(c)) { + return classType(Dict.class); + } + throw new InvalidStarlarkValueException(c); + } + + /** + * The ordering relation over (some) Starlark values. + * + *

Starlark values are ordered as follows. + * + *

    + *
  • {@code False < True}. + *
  • int values are ordered according to mathematical tradition. + *
  • float values are ordered according to IEEE 754, with the exception of NaN values: all NaN + * values compare equal to each other and greater than +Inf. The zero values 0.0 and -0.0 + * compare equal. + *
  • int and float values may be compared. The comparison is mathematically exact, even if + * neither argument may be exactly converted to the type of the other. This is the only + * permitted case of comparisons between values of different types. NaN values compare + * greater than all integers. + *
  • Strings are ordered lexicographically by their elements (chars). So too are lists and + * tuples, though lists are not comparable with tuples. + *
  • If x implements Comparable, its {@code compareTo(y)} method may be called to determine + * the comparison if x and y have the same {@link #type}, though not necessary the same Java + * class. + *
  • Ordered comparison of any other values is an error (ClassCastException). + *
+ * + *

This method defines a strict weak ordering that is consistent with {@link Object#equals}. + */ + public static final Ordering ORDERING = + new Ordering() { + @Override + public int compare(Object x, Object y) { + return compareUnchecked(x, y); + } + }; + + /** + * Defines the strict weak ordering of Starlark values used for sorting and the comparison + * operators. Throws ClassCastException on failure. + */ + static int compareUnchecked(Object x, Object y) { + if (sameType(x, y)) { + // Ordered? e.g. string, int, bool, float. + if (x instanceof Comparable) { + @SuppressWarnings("unchecked") + Comparable xcomp = (Comparable) x; + return xcomp.compareTo(y); + } + + } else { + // different types + + if (x instanceof StarlarkFloat && y instanceof StarlarkInt) { + // float < int + double xf = ((StarlarkFloat) x).toDouble(); + return Double.isNaN(xf) ? +1 : -StarlarkInt.compareIntAndDouble((StarlarkInt) y, xf); + } else if (x instanceof StarlarkInt && y instanceof StarlarkFloat) { + // int < float + double yf = ((StarlarkFloat) y).toDouble(); + return Double.isNaN(yf) ? -1 : StarlarkInt.compareIntAndDouble((StarlarkInt) x, yf); + } + } + + throw new ClassCastException( + String.format("unsupported comparison: %s <=> %s", Starlark.type(x), Starlark.type(y))); + } + + private static boolean sameType(Object x, Object y) { + return x.getClass() == y.getClass() || Starlark.type(x).equals(Starlark.type(y)); + } + + /** Returns the string form of a value as if by the Starlark expression {@code str(x)}. */ + public static String str(Object x, StarlarkSemantics semantics) { + return new Printer().str(x, semantics).toString(); + } + + /** Returns the string form of a value as if by the Starlark expression {@code repr(x)}. */ + public static String repr(Object x) { + return new Printer().repr(x).toString(); + } + + /** Returns a string formatted as if by the Starlark expression {@code pattern % arguments}. */ + public static String format(StarlarkSemantics semantics, String pattern, Object... arguments) { + Printer pr = new Printer(); + Printer.format(pr, semantics, pattern, arguments); + return pr.toString(); + } + + /** Returns a string formatted as if by the Starlark expression {@code pattern % arguments}. */ + public static String formatWithList( + StarlarkSemantics semantics, String pattern, List arguments) { + Printer pr = new Printer(); + Printer.formatWithList(pr, semantics, pattern, arguments); + return pr.toString(); + } + + /** + * Returns a Starlark doc string with each line trimmed and dedented to the minimal common + * indentation level (except for the first line, which is always fully trimmed), and with leading + * and trailing empty lines removed, following the PEP-257 algorithm. See + * https://peps.python.org/pep-0257/#handling-docstring-indentation + * + *

For whitespace trimming, we use the same definition of whitespace as the Starlark {@code + * string.strip} method. + * + *

Following PEP-257, we expand tabs in the doc string with tab size 8 before dedenting. + * Starlark does not use tabs for indentation, but Starlark string values may contain tabs, so we + * choose to expand them for consistency with Python. + * + *

The intent is to turn documentation strings like + * + * 

+   *     """Heading
+   *
+   *     Details paragraph
+   *     """
+   *
+ * + * and + * + * 
+   *     """
+   *     Heading
+   *
+   *     Details paragraph
+   *     """
+   *
+ * + * into the desired "Heading\n\nDetails paragraph" form, and avoid the risk of documentation + * processors interpreting indented parts of the original string as special formatting (e.g. code + * blocks in the case of Markdown). + */ + public static String trimDocString(String docString) { + ImmutableList lines = expandTabs(docString, 8).lines().collect(toImmutableList()); + if (lines.isEmpty()) { + return ""; + } + // First line is special: we fully strip it and ignore it for leading spaces calculation + String firstLineTrimmed = StringModule.INSTANCE.strip(lines.get(0), NONE); + Iterable subsequentLines = Iterables.skip(lines, 1); + int minLeadingSpaces = Integer.MAX_VALUE; + for (String line : subsequentLines) { + String strippedLeading = StringModule.INSTANCE.lstrip(line, NONE); + if (!strippedLeading.isEmpty()) { + int leadingSpaces = line.length() - strippedLeading.length(); + minLeadingSpaces = min(leadingSpaces, minLeadingSpaces); + } + } + if (minLeadingSpaces == Integer.MAX_VALUE) { + minLeadingSpaces = 0; + } + + StringBuilder result = new StringBuilder(); + result.append(firstLineTrimmed); + for (String line : subsequentLines) { + // Length check ensures we ignore leading empty lines + if (result.length() > 0) { + result.append("\n"); + } + if (line.length() > minLeadingSpaces) { + result.append(StringModule.INSTANCE.rstrip(line.substring(minLeadingSpaces), NONE)); + } + } + // Remove trailing empty lines + return StringModule.INSTANCE.rstrip(result.toString(), NONE); + } + + /** + * Expands tab characters to one or more spaces, producing the same indentation level at any given + * point on any given line as would be expected when rendering the string with a given tab size; a + * Java port of Python's {@code str.expandtabs}. + */ + static String expandTabs(String line, int tabSize) { + if (!line.contains("\t")) { + // Don't alloc in the fast case. + return line; + } + checkArgument(tabSize > 0); + StringBuilder result = new StringBuilder(); + int col = 0; + for (int i = 0; i < line.length(); i++) { + char c = line.charAt(i); + switch (c) { + case '\n': + case '\r': + result.append(c); + col = 0; + break; + case '\t': + int spaces = tabSize - col % tabSize; + for (int j = 0; j < spaces; j++) { + result.append(' '); + } + col += spaces; + break; + default: + result.append(c); + col++; + } + } + return result.toString(); + } + + /** Returns a slice of a sequence as if by the Starlark operation {@code x[start:stop:step]}. */ + public static Object slice( + Mutability mu, Object x, Object startObj, Object stopObj, Object stepObj) + throws EvalException { + int n; + if (x instanceof String) { + n = ((String) x).length(); + } else if (x instanceof Sequence) { + n = ((Sequence) x).size(); + } else { + throw errorf("invalid slice operand: %s", type(x)); + } + + int start; + int stop; + int step; + + // step + if (stepObj == NONE) { + step = 1; + } else { + step = toInt(stepObj, "slice step"); + if (step == 0) { + throw errorf("slice step cannot be zero"); + } + } + + // start, stop + if (step > 0) { + // positive stride: default indices are [0:n]. + if (startObj == NONE) { + start = 0; + } else { + start = EvalUtils.toIndex(toInt(startObj, "start index"), n); + } + + if (stopObj == NONE) { + stop = n; + } else { + stop = EvalUtils.toIndex(toInt(stopObj, "stop index"), n); + } + + if (stop < start) { + stop = start; // => empty result + } + + } else { + // negative stride: default indices are effectively [n-1:-1], + // though to get this effect using explicit indices requires + // [n-1:-1-n:-1] because of the treatment of negative values. + if (startObj == NONE) { + start = n - 1; + } else { + start = toInt(startObj, "start index"); + if (start < 0) { + start += n; + } + if (start >= n) { + start = n - 1; + } + } + + if (stopObj == NONE) { + stop = -1; + } else { + stop = toInt(stopObj, "stop index"); + if (stop < 0) { + stop += n; + } + if (stop < -1) { + stop = -1; + } + } + + if (start < stop) { + start = stop; // => empty result + } + } + + // slice operation + if (x instanceof String) { + return StringModule.slice((String) x, start, stop, step); + } else { + return ((Sequence) x).getSlice(mu, start, stop, step); + } + } + + /** + * Returns the signed 32-bit value of a Starlark int. Throws an exception including {@code what} + * if x is not a Starlark int or its value is not exactly representable as a Java int. + * + * @throws IllegalArgumentException if x is an Integer, which is not a Starlark value. + */ + public static int toInt(Object x, String what) throws EvalException { + if (x instanceof StarlarkInt) { + return ((StarlarkInt) x).toInt(what); + } + if (x instanceof Integer) { + throw new IllegalArgumentException("Integer is not a legal Starlark value"); + } + throw errorf("got %s for %s, want int", type(x), what); + } + + /** + * Calls the function-like value {@code fn} in the specified thread, passing it the given + * positional and named arguments, as if by the Starlark expression {@code fn(*args, **kwargs)}. + * + *

See also {@link #fastcall}. + */ + public static Object call( + StarlarkThread thread, Object fn, List args, Map kwargs) + throws EvalException, InterruptedException { + Object[] named = new Object[2 * kwargs.size()]; + int i = 0; + for (Map.Entry e : kwargs.entrySet()) { + named[i++] = e.getKey(); + named[i++] = Starlark.checkValid(e.getValue()); + } + return fastcall(thread, fn, args.toArray(), named); + } + + /** + * Calls the function-like value {@code fn} in the specified thread, passing it the given + * positional and named arguments in the "fastcall" array representation. + * + *

The caller must not subsequently modify or even inspect the two arrays. + * + *

If the call throws an unchecked throwable, regardless of whether it originates in a + * user-defined built-in function or a bug in the interpreter itself, the throwable is wrapped by + * {@link UncheckedEvalException} (for {@link RuntimeException}) or {@link UncheckedEvalError} + * (for {@link Error}). The {@linkplain Throwable#getStackTrace stack trace} will reflect the + * Starlark call stack rather than the Java call stack. The original throwable (and the Java call + * stack) may be retrieved using {@link Throwable#getCause}. + */ + public static Object fastcall( + StarlarkThread thread, Object fn, Object[] positional, Object[] named) + throws EvalException, InterruptedException { + StarlarkCallable callable; + if (fn instanceof StarlarkCallable) { + callable = (StarlarkCallable) fn; + } else { + // @StarlarkMethod(selfCall)? + MethodDescriptor desc = + CallUtils.getSelfCallMethodDescriptor(thread.getSemantics(), fn.getClass()); + if (desc == null) { + throw errorf("'%s' object is not callable", type(fn)); + } + callable = new BuiltinFunction(fn, desc.getName(), desc); + } + + thread.push(callable); + try { + return callable.fastcall(thread, positional, named); + } catch (UncheckedEvalException | UncheckedEvalError ex) { + throw ex; // already wrapped + } catch (RuntimeException ex) { + throw new UncheckedEvalException(ex, thread); + } catch (Error ex) { + throw new UncheckedEvalError(ex, thread); + } catch (EvalException ex) { + // If this exception was newly thrown, set its stack. + throw ex.ensureStack(thread); + } finally { + thread.pop(); + } + } + + /** + * Decorates a {@link RuntimeException} with its Starlark stack, to help maintainers locate + * problematic source expressions. + * + *

The original exception can be retrieved using {@link #getCause}. + */ + public static final class UncheckedEvalException extends RuntimeException { + + private UncheckedEvalException(RuntimeException cause, StarlarkThread thread) { + super(createUncheckedEvalMessage(cause, thread), cause); + thread.fillInStackTrace(this); + } + } + + /** + * Decorates an {@link Error} with its Starlark stack, to help maintainers locate problematic + * source expressions. + * + *

The original exception can be retrieved using {@link #getCause}. + */ + public static final class UncheckedEvalError extends Error { + + private UncheckedEvalError(Error cause, StarlarkThread thread) { + super(createUncheckedEvalMessage(cause, thread), cause); + thread.fillInStackTrace(this); + } + } + + private static String createUncheckedEvalMessage(Throwable cause, StarlarkThread thread) { + String msg = cause.getClass().getSimpleName() + " thrown during Starlark evaluation"; + String context = thread.getContextDescription(); + return isNullOrEmpty(context) ? msg : msg + " (" + context + ")"; + } + + /** + * Returns a new EvalException with no location and an error message produced by Java-style string + * formatting ({@code String.format(format, args)}). Use {@code errorf("%s", msg)} to produce an + * error message from a non-constant expression {@code msg}. + */ + @FormatMethod + @CheckReturnValue // don't forget to throw it + public static EvalException errorf(String format, Object... args) { + return new EvalException(String.format(format, args)); + } + + // --- methods related to attributes (fields and methods) --- + + /** + * Reports whether the value {@code x} has a field or method of the given name, as if by the + * Starlark expression {@code hasattr(x, name)}. + */ + public static boolean hasattr(StarlarkSemantics semantics, Object x, String name) + throws EvalException { + return (x instanceof Structure && ((Structure) x).getValue(name) != null) + || CallUtils.getAnnotatedMethods(semantics, x.getClass()).containsKey(name); + } + + /** + * Returns the named field or method of value {@code x}, as if by the Starlark expression {@code + * getattr(x, name, defaultValue)}. If the value has no such attribute, getattr returns {@code + * defaultValue} if non-null, or throws an EvalException otherwise. + */ + public static Object getattr( + Mutability mu, + StarlarkSemantics semantics, + Object x, + String name, + @Nullable Object defaultValue) + throws EvalException, InterruptedException { + // StarlarkMethod-annotated field or method? + MethodDescriptor method = CallUtils.getAnnotatedMethods(semantics, x.getClass()).get(name); + if (method != null) { + if (method.isStructField()) { + return method.callField(x, semantics, mu); + } else { + return new BuiltinFunction(x, name, method); + } + } + + // user-defined field? + if (x instanceof Structure struct) { + Object field = struct.getValue(semantics, name); + if (field != null) { + return Starlark.checkValid(field); + } + + if (defaultValue != null) { + return defaultValue; + } + + String error = struct.getErrorMessageForUnknownField(name); + if (error != null) { + throw Starlark.errorf("%s", error); + } + + } else if (defaultValue != null) { + return defaultValue; + } + + throw Starlark.errorf( + "'%s' value has no field or method '%s'%s", + Starlark.type(x), name, SpellChecker.didYouMean(name, dir(mu, semantics, x))); + } + + /** + * Returns a new sorted list containing the names of the Starlark-accessible fields and methods of + * the specified value, as if by the Starlark expression {@code dir(x)}. + */ + public static StarlarkList dir(Mutability mu, StarlarkSemantics semantics, Object x) { + // Order the fields alphabetically. + Set fields = new TreeSet<>(); + if (x instanceof Structure) { + fields.addAll(((Structure) x).getFieldNames()); + } + fields.addAll(CallUtils.getAnnotatedMethods(semantics, x.getClass()).keySet()); + return StarlarkList.copyOf(mu, fields); + } + + // --- methods related to StarlarkMethod-annotated classes --- + + /** + * Returns the value of the named field of Starlark value {@code x}, as defined by a Java method + * with a {@code StarlarkMethod(structField=true)} annotation. + * + *

Most callers should use {@link #getattr} instead. + */ + public static Object getAnnotatedField(StarlarkSemantics semantics, Object x, String name) + throws EvalException, InterruptedException { + return CallUtils.getAnnotatedField(semantics, x, name); + } + + /** + * Returns the names of the fields of Starlark value {@code x}, as defined by Java methods with + * {@code StarlarkMethod(structField=true)} annotations under the specified semantics. + * + *

Most callers should use {@link #dir} instead. + */ + public static ImmutableSet getAnnotatedFieldNames(StarlarkSemantics semantics, Object x) { + return CallUtils.getAnnotatedFieldNames(semantics, x); + } + + /** + * Returns a map of Java methods and corresponding StarlarkMethod annotations for each annotated + * Java method of the specified class. Elements are ordered by Java method name, which is not + * necessarily the same as the Starlark attribute name. The set of enabled methods is determined + * by {@link StarlarkSemantics#DEFAULT}. Excludes the {@code selfCall} method, if any. + * + *

Most callers should use {@link #dir} and {@link #getattr} instead. + */ + // TODO(adonovan): move to StarlarkAnnotations; it's a static property of the annotations. + public static ImmutableMap getMethodAnnotations(Class clazz) { + ImmutableMap.Builder result = ImmutableMap.builder(); + for (MethodDescriptor desc : + CallUtils.getAnnotatedMethods(StarlarkSemantics.DEFAULT, clazz).values()) { + result.put(desc.getMethod(), desc.getAnnotation()); + } + return result.build(); + } + + /** + * Returns the {@code StarlarkMethod(selfCall=true)}-annotated Java method of the specified Java + * class that is called when Starlark calls an instance of that class like a function. It returns + * null if no such method exists. + */ + @Nullable + public static Method getSelfCallMethod(StarlarkSemantics semantics, Class clazz) { + return CallUtils.getSelfCallMethod(semantics, clazz); + } + + /** Equivalent to {@code addMethods(env, v, StarlarkSemantics.DEFAULT)}. */ + public static void addMethods(ImmutableMap.Builder env, Object v) { + addMethods(env, v, StarlarkSemantics.DEFAULT); + } + + /** + * Adds to the environment {@code env} all Starlark methods of value {@code v}, filtered by the + * given semantics. Starlark methods are Java methods of {@code v} with a {@link StarlarkMethod} + * annotation whose {@code structField} and {@code selfCall} flags are both false. + * + * @throws IllegalArgumentException if any method annotation's {@link StarlarkMethod#structField} + * flag is true. + */ + public static void addMethods( + ImmutableMap.Builder env, Object v, StarlarkSemantics semantics) { + Class cls = v.getClass(); + // TODO(adonovan): rather than silently skip the selfCall method, reject it. + for (Map.Entry e : + CallUtils.getAnnotatedMethods(semantics, cls).entrySet()) { + String name = e.getKey(); + + // We cannot accept fields, as they are inherently problematic: + // what if the Java method call fails, or gets interrupted? + if (e.getValue().isStructField()) { + throw new IllegalArgumentException( + String.format("addMethods(%s): method %s has structField=true", cls.getName(), name)); + } + + // We use the 2-arg (desc=null) BuiltinFunction constructor instead of passing + // the descriptor that CallUtils.getAnnotatedMethod would return, + // because most calls to addMethods implicitly pass StarlarkSemantics.DEFAULT, + // which is probably the wrong semantics for the later call. + // + // The effect is that the default semantics determine which method names are + // statically available in the environment, but the thread's semantics determine + // the dynamic behavior of the method call; this includes a run-time check for + // whether the method was disabled by the semantics. + env.put(name, new BuiltinFunction(v, name)); + } + } + + /** + * Parses the input as a file, resolves it in the specified module environment, compiles it, and + * executes it in the specified thread. On success it returns None, unless the file's final + * statement is an expression, in which case its value is returned. + * + * @throws SyntaxError.Exception if there were (static) scanner, parser, or resolver errors. + * @throws EvalException if there was a (dynamic) evaluation error. + * @throws InterruptedException if the Java thread was interrupted during evaluation. + */ + public static Object execFile( + ParserInput input, FileOptions options, Module module, StarlarkThread thread) + throws SyntaxError.Exception, EvalException, InterruptedException { + StarlarkFile file = StarlarkFile.parse(input, options); + Program prog = Program.compileFile(file, module); + return execFileProgram(prog, module, thread); + } + + /** Variant of {@link #execFile} that creates a module for the given predeclared environment. */ + // TODO(adonovan): is this needed? + public static Object execFile( + ParserInput input, + FileOptions options, + Map predeclared, + StarlarkThread thread) + throws SyntaxError.Exception, EvalException, InterruptedException { + Module module = Module.withPredeclared(thread.getSemantics(), predeclared); + return execFile(input, options, module, thread); + } + + /** + * Executes a compiled Starlark file (as obtained from {@link Program#compileFile}) in the given + * StarlarkThread. On success it returns None, unless the file's final statement is an expression, + * in which case its value is returned. + * + * @throws EvalException if there was a (dynamic) evaluation error. + * @throws InterruptedException if the Java thread was interrupted during evaluation. + */ + public static Object execFileProgram(Program prog, Module module, StarlarkThread thread) + throws EvalException, InterruptedException { + Resolver.Function rfn = prog.getResolvedFunction(); + + // A given Module may be passed to execFileProgram multiple times in sequence, + // for different compiled Programs. (This happens in the REPL, and in + // EvaluationTestCase scenarios. It is not true of the go.starlark.net + // implementation, and it complicates things significantly. + // It would be nice to stop doing that.) + // + // Therefore StarlarkFunctions from different Programs (files) but initializing + // the same Module need different mappings from the Program's numbering of + // globals to the Module's numbering of globals, and to access a global requires + // two array lookups. + int[] globalIndex = module.getIndicesOfGlobals(rfn.getGlobals()); + + if (module.getDocumentation() == null) { + String documentation = rfn.getDocumentation(); + if (documentation != null) { + module.setDocumentation(Starlark.trimDocString(documentation)); + } + } + + StarlarkFunction toplevel = + new StarlarkFunction( + rfn, + module, + globalIndex, + /* defaultValues= */ Tuple.empty(), + /* freevars= */ Tuple.empty(), + thread.getNextIdentityToken()); + return Starlark.fastcall(thread, toplevel, EMPTY, EMPTY); + } + + private static final Object[] EMPTY = {}; + + /** + * Parses the input as an expression, resolves it in the specified module environment, compiles + * it, evaluates it, and returns its value. + * + * @throws SyntaxError.Exception if there were (static) scanner, parser, or resolver errors. + * @throws EvalException if there was a (dynamic) evaluation error. + * @throws InterruptedException if the Java thread was interrupted during evaluation. + */ + public static Object eval( + ParserInput input, FileOptions options, Module module, StarlarkThread thread) + throws SyntaxError.Exception, EvalException, InterruptedException { + StarlarkFunction fn = newExprFunction(input, options, module, thread.getNextIdentityToken()); + return Starlark.fastcall(thread, fn, EMPTY, EMPTY); + } + + /** Variant of {@link #eval} that creates a module for the given predeclared environment. */ + // TODO(adonovan): is this needed? + public static Object eval( + ParserInput input, + FileOptions options, + Map predeclared, + StarlarkThread thread) + throws SyntaxError.Exception, EvalException, InterruptedException { + Module module = Module.withPredeclared(thread.getSemantics(), predeclared); + return eval(input, options, module, thread); + } + + /** + * Parses the input as an expression, resolves it in the specified module environment, and returns + * a callable no-argument Starlark function value that computes and returns the value of the + * expression. + * + * @throws SyntaxError.Exception if there were scanner, parser, or resolver errors. + */ + private static StarlarkFunction newExprFunction( + ParserInput input, + FileOptions options, + Module module, + SymbolGenerator.Symbol referenceIdentity) + throws SyntaxError.Exception { + Expression expr = Expression.parse(input); + Program prog = Program.compileExpr(expr, module, options); + Resolver.Function rfn = prog.getResolvedFunction(); + int[] globalIndex = module.getIndicesOfGlobals(rfn.getGlobals()); // see execFileProgram + return new StarlarkFunction( + rfn, + module, + globalIndex, + /* defaultValues= */ Tuple.empty(), + /* freevars= */ Tuple.empty(), + referenceIdentity); + } + + /** + * Starts the CPU profiler with the specified sampling period, writing a pprof profile to {@code + * out}. All running Starlark threads are profiled. May be called concurrent with Starlark + * execution. + * + * @throws IllegalStateException exception if the Starlark profiler is already running or if the + * operating system's profiling resources for this process are already in use. + */ + public static boolean startCpuProfile(OutputStream out, Duration period) { + return CpuProfiler.start(out, period); + } + + /** + * Stops the profiler and waits for the log to be written. Throws an unchecked exception if the + * profiler was not already started by a prior call to {@link #startCpuProfile}. + */ + public static void stopCpuProfile() throws IOException { + CpuProfiler.stop(); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkCallable.java b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkCallable.java new file mode 100644 index 000000000..fa6402953 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkCallable.java @@ -0,0 +1,99 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.collect.Maps; +import java.util.LinkedHashMap; +import net.starlark.java.syntax.Location; + +/** + * The StarlarkCallable interface is implemented by all Starlark values that may be called from + * Starlark like a function, including built-in functions and methods, Starlark functions, and + * application-defined objects (such as rules, aspects, and providers in Bazel). + * + *

It defines two methods: {@code fastcall}, for performance, or {@code call} for convenience. By + * default, {@code fastcall} delegates to {@code call}, and call throws an exception, so an + * implementer may override either one. + */ +public interface StarlarkCallable extends StarlarkValue { + + /** + * Defines the "convenient" implementation of function calling for a callable value. + * + *

Do not call this function directly. Use the {@link Starlark#call} function to make a call, + * as it handles necessary book-keeping such as maintenance of the call stack, exception handling, + * and so on. + * + *

The default implementation throws an EvalException. + * + *

See {@link Starlark#fastcall} for basic information about function calls. + * + * @param thread the StarlarkThread in which the function is called + * @param args a tuple of the arguments passed by position + * @param kwargs a new, mutable dict of the arguments passed by keyword. Iteration order is + * determined by keyword order in the call expression. + */ + default Object call(StarlarkThread thread, Tuple args, Dict kwargs) + throws EvalException, InterruptedException { + throw Starlark.errorf("function %s not implemented", getName()); + } + + /** + * Defines the "fast" implementation of function calling for a callable value. + * + *

Do not call this function directly. Use the {@link Starlark#call} or {@link + * Starlark#fastcall} function to make a call, as it handles necessary book-keeping such as + * maintenance of the call stack, exception handling, and so on. + * + *

The fastcall implementation takes ownership of the two arrays, and may retain them + * indefinitely or modify them. The caller must not modify or even access the two arrays after + * making the call. + * + *

This method defines the low-level or "fast" calling convention. A more convenient interface + * is provided by the {@link #call} method, which provides a signature analogous to {@code def + * f(*args, **kwargs)}, or possibly the "self-call" feature of the {@link StarlarkMethod#selfCall} + * annotation mechanism. + * + *

The default implementation forwards the call to {@code call}, after rejecting any duplicate + * named arguments. Other implementations of this method should similarly reject duplicates. + * + *

See {@link Starlark#fastcall} for basic information about function calls. + * + * @param thread the StarlarkThread in which the function is called + * @param positional a list of positional arguments + * @param named a list of named arguments, as alternating Strings/Objects. May contain dups. + */ + default Object fastcall(StarlarkThread thread, Object[] positional, Object[] named) + throws EvalException, InterruptedException { + LinkedHashMap kwargs = Maps.newLinkedHashMapWithExpectedSize(named.length >> 1); + for (int i = 0; i < named.length; i += 2) { + if (kwargs.put((String) named[i], named[i + 1]) != null) { + throw Starlark.errorf("%s got multiple values for parameter '%s'", this, named[i]); + } + } + return call(thread, Tuple.of(positional), Dict.wrap(thread.mutability(), kwargs)); + } + + /** Returns the form this callable value should take in a stack trace. */ + String getName(); + + /** + * Returns the location of the definition of this callable value, or BUILTIN if it was not defined + * in Starlark code. + */ + default Location getLocation() { + return Location.BUILTIN; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkFloat.java b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkFloat.java new file mode 100644 index 000000000..a92f99ab0 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkFloat.java @@ -0,0 +1,290 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import java.math.BigInteger; +import java.util.Locale; +import net.starlark.java.annot.StarlarkBuiltin; + +/** The Starlark float data type. */ +@StarlarkBuiltin( + name = "float", + category = "core", + doc = "The type of floating-point numbers in Starlark.") +public final class StarlarkFloat implements StarlarkValue, Comparable { + + private final double v; + + private StarlarkFloat(double v) { + this.v = v; + } + + /** Returns the Starlark float value that represents x. */ + public static StarlarkFloat of(double v) { + return new StarlarkFloat(v); + } + + /** Returns the value of this float. */ + public double toDouble() { + return v; + } + + @Override + public String toString() { + return format(v, 'g'); + } + + @Override + public void repr(Printer printer) { + printer.append(toString()); + } + + @Override + public boolean isImmutable() { + return true; + } + + @Override + public boolean truth() { + return this.v != 0.0; + } + + /** + * Defines a total order over float values. Positive and negative zero values compare equal. NaN + * compares equal to itself and greater than +Inf. + */ + @Override + public int compareTo(StarlarkFloat that) { + double x = this.v; + double y = that.v; + if (x > y) { + return +1; + } else if (x < y) { + return -1; + } else if (x == y) { + return 0; // 0.0 == -0.0 + } + + // At least one operand is NaN. + // Canonicalize NaNs using doubleToLongBits and compare bits. + long xbits = Double.doubleToLongBits(x); + long ybits = Double.doubleToLongBits(y); + return Long.compare(xbits, ybits); // NaN > non-NaN + } + + @Override + public int hashCode() { + // Equal float and int values must yield the same hash. + if (Double.isFinite(v) && v == Math.rint(v)) { + return StarlarkInt.ofFiniteDouble(v).hashCode(); + } + + // For non-integral values we can use a cheaper hash. + // Hashing the bits is consistent with equals + // because v is neither 0.0 nor -0.0. + long bits = Double.doubleToLongBits(v); // canonicalizes NaNs + return (int) (bits ^ (bits >>> 32)); + } + + @Override + public boolean equals(Object that) { + return (that instanceof StarlarkFloat && equal(this.v, ((StarlarkFloat) that).v)) + || (that instanceof StarlarkInt && StarlarkInt.intEqualsFloat((StarlarkInt) that, this)); + } + + // equal is an equivalence relation consistent with hashCode and compareTo. + private static boolean equal(double x, double y) { + return x == y || (Double.isNaN(x) && Double.isNaN(y)); + } + + // Performs printf-style string conversion of a double value v. + // conv is one of [efgEFG]. + static String format(double v, char conv) { + if (!Double.isFinite(v)) { + if (v == Double.POSITIVE_INFINITY) { + return "+inf"; + } else if (v == Double.NEGATIVE_INFINITY) { + return "-inf"; + } else { + return "nan"; + } + } + + String s; + switch (conv) { + case 'e': + s = String.format(Locale.US, "%e", v); + break; + case 'E': + s = String.format(Locale.US, "%E", v); + break; + case 'f': + case 'F': // an alias + s = String.format(Locale.US, "%f", v); + break; + case 'g': + s = String.format(Locale.US, "%.17g", v); // use DBL_DECIMAL_DIG places + break; + case 'G': + s = String.format(Locale.US, "%.17G", v); + break; + default: + throw new IllegalArgumentException("unsupported conversion: " + conv); + } + + // %g is the default format used by str. + // It always includes a '.' or an 'e', to make clear that + // the value is a float, not an int. + // + // TODO(adonovan): round the value to the minimal precision required + // to avoid ambiguity. This requires computing the decimal digit + // strings of the adjacent floating-point values and then taking the + // shortest prefix sufficient to distinguish v from them, or using a + // more sophisticated algorithm such as Florian Loitsch's Grisu3 or + // Ulf Adams' Ryu. (Is there an easy way to compute the required + // precision without materializing the digits? If so we could delegate + // to format("%*g", prec, v).) + // + // For now, we just clean up the output of Java's %.17g implementation, + // which is unambiguous, but may yield unnecessarily long digit strings + // such as 1000000000000.0. + if (conv == 'g' || conv == 'G') { + int e = s.indexOf(conv == 'g' ? 'e' : 'E'); + if (e < 0) { + int dot = s.indexOf('.'); + if (dot < 0) { + // Ensure result always has a decimal point if no exponent. + // "123" -> "123.0" + s += ".0"; + } else { + // Remove trailing zeros after decimal point. + // "1.110" => "1.11" + // "1.000" => "1.0" + int i; + for (i = s.length() - 1; i > dot + 1 && s.charAt(i) == '0'; i--) {} + s = s.substring(0, i + 1); + } + } else { + // Remove trailing zeros from mantissa. + // "1.23000e+45" => "1.23e+45" + // "1.00000e+45" => "1e+45" + int i; + for (i = e - 1; s.charAt(i) == '0'; i--) {} + if (s.charAt(i) == '.') { + i--; + } + if (i < e - 1) { + s = + new StringBuilder(i + 1 + s.length() - e) + .append(s, 0, i + 1) // "1.23" + .append(s, e, s.length()) // "e+45" + .toString(); + } + } + } + + return s; + } + + /** Returns x // y (floor of division). */ + static StarlarkFloat floordiv(double x, double y) throws EvalException { + if (y == 0.0) { + throw Starlark.errorf("integer division by zero"); + } + return StarlarkFloat.of(Math.floor(x / y)); + } + + /** Returns x / y (floating-point division). */ + static StarlarkFloat div(double x, double y) throws EvalException { + if (y == 0.0) { + throw Starlark.errorf("floating-point division by zero"); + } + return StarlarkFloat.of(x / y); + } + + /** Returns x % y (floating-point remainder). */ + static StarlarkFloat mod(double x, double y) throws EvalException { + if (y == 0.0) { + throw Starlark.errorf("floating-point modulo by zero"); + } + // In Starlark, the sign of the result is the sign of the divisor. + double z = x % y; + if ((x < 0) != (y < 0) && z != 0) { + z += y; + } + return StarlarkFloat.of(z); + } + + /** + * Returns the Starlark int value closest to x, truncating towards zero. + * + * @throws IllegalArgumentException if x is not finite (NaN or ±Inf). + */ + static StarlarkInt finiteDoubleToIntExact(double x) { + // small value? + if (Long.MIN_VALUE <= x && x <= Long.MAX_VALUE) { + return StarlarkInt.of((long) x); + } + + // Shift must be positive, because we just handled all small values. + int shift = getShift(x); + if (shift <= 0) { + throw new IllegalStateException("non-positive shift"); + } + + // Shift mantissa by exponent. + long mantissa = getMantissa(x); + return StarlarkInt.of(BigInteger.valueOf(mantissa).shiftLeft(shift)); + } + + private static final int EXPONENT_MASK = (1 << 11) - 1; + + // Returns the effective signed mantissa of x. + // Precondition: x is finite. + static long getMantissa(double x) { + long bits = Double.doubleToRawLongBits(x); + long mantissa = bits & ((1L << 52) - 1); + int exp = ((int) (bits >>> 52)) & EXPONENT_MASK; + switch (exp) { + case 0: // denormal + break; + case EXPONENT_MASK: + throw new IllegalArgumentException("not finite: " + x); + default: // normal + mantissa |= 1L << 52; + break; + } + return x < 0 ? -mantissa : mantissa; + } + + // Returns the effective left (+) or right (-) shift required of the value returned by + // getMantissa(x). + // Precondition: x is finite. + static int getShift(double x) { + long bits = Double.doubleToRawLongBits(x); + int exp = ((int) (bits >>> 52)) & EXPONENT_MASK; + switch (exp) { + case 0: // denormal + exp -= 1022; + break; + case EXPONENT_MASK: + throw new IllegalArgumentException("not finite: " + x); + default: // normal + exp -= 1023; + break; + } + return exp - 52; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkFunction.java b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkFunction.java new file mode 100644 index 000000000..37d7ab020 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkFunction.java @@ -0,0 +1,472 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.eval; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import javax.annotation.Nullable; +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.spelling.SpellChecker; +import net.starlark.java.syntax.Location; +import net.starlark.java.syntax.Resolver; + +/** A StarlarkFunction is a function value created by a Starlark {@code def} statement. */ +@StarlarkBuiltin( + name = "function", + category = "core", + doc = "The type of functions declared in Starlark.") +public final class StarlarkFunction implements StarlarkCallable { + + final Resolver.Function rfn; + private final Module module; // a function closes over its defining module + + // Index in Module.globals of ith Program global (Resolver.Binding(GLOBAL).index). + // See explanation at Starlark.execFileProgram. + final int[] globalIndex; + + // Default values of optional parameters. + // Indices correspond to the subsequence of parameters after the initial + // required parameters and before *args/**kwargs. + // Contain MANDATORY for the required keyword-only parameters. + private final Tuple defaultValues; + + // Cells (shared locals) of enclosing functions. + // Indexed by Resolver.Binding(FREE).index values. + private final Tuple freevars; + + // A stable identifier for this function instance. + // + // This may be mutated by export. + private SymbolGenerator.Symbol token; + + StarlarkFunction( + Resolver.Function rfn, + Module module, + int[] globalIndex, + Tuple defaultValues, + Tuple freevars, + SymbolGenerator.Symbol token) { + this.rfn = rfn; + this.module = module; + this.globalIndex = globalIndex; + this.defaultValues = defaultValues; + this.freevars = freevars; + this.token = token; + } + + // Sets a global variable, given its index in this function's compiled Program. + void setGlobal(int progIndex, Object value) { + module.setGlobalByIndex(globalIndex[progIndex], value); + } + + // Gets the value of a global variable, given its index in this function's compiled Program. + @Nullable + Object getGlobal(int progIndex) { + return module.getGlobalByIndex(globalIndex[progIndex]); + } + + boolean isToplevel() { + return rfn.isToplevel(); + } + + /** Whether this function is defined at the top level of a file. */ + public boolean isGlobal() { + return module.getGlobal(getName()) == this; + } + + // TODO(adonovan): many functions would be simpler if + // parameterNames excluded the *args and **kwargs parameters, + // (whose names are immaterial to the callee anyway). Do that. + // Also, reject getDefaultValue for varargs and kwargs. + + /** + * Returns the default value of the ith parameter ({@code 0 <= i < getParameterNames().size()}), + * or null if the parameter is required. Residual parameters, if any, are always last, and have no + * default value. + */ + @Nullable + public Object getDefaultValue(int i) { + if (i < 0 || i >= rfn.getParameters().size()) { + throw new IndexOutOfBoundsException(); + } + int nparams = + rfn.getParameters().size() - (rfn.hasKwargs() ? 1 : 0) - (rfn.hasVarargs() ? 1 : 0); + int prefix = nparams - defaultValues.size(); + if (i < prefix) { + return null; // implicit prefix of mandatory parameters + } + if (i < nparams) { + Object v = defaultValues.get(i - prefix); + return v == MANDATORY ? null : v; + } + return null; // *args or *kwargs + } + + /** + * Returns the names of this function's parameters. + * + *

The first {@code getNumOrdinaryParameters()} parameters in the returned list are ordinary + * (non-residual, non-keyword-only); the following {@code getNumKeywordOnlyParameters()} are + * keyword-only; and the residual {@code *args} and {@code **kwargs} parameters, if any, are + * always last. + */ + public ImmutableList getParameterNames() { + return rfn.getParameterNames(); + } + + /** Returns the number of ordinary (non-residual, non-keyword-only) parameters. */ + public int getNumOrdinaryParameters() { + return rfn.getParameters().size() + - (rfn.hasKwargs() ? 1 : 0) + - (rfn.hasVarargs() ? 1 : 0) + - rfn.numKeywordOnlyParams(); + } + + /** Returns the number of non-residual keyword-only parameters. */ + public int getNumKeywordOnlyParameters() { + return rfn.numKeywordOnlyParams(); + } + + /** + * Reports whether this function has a residual positional arguments parameter, {@code def + * f(*args)}. + */ + public boolean hasVarargs() { + return rfn.hasVarargs(); + } + + /** + * Reports whether this function has a residual keyword arguments parameter, {@code def + * f(**kwargs)}. + */ + public boolean hasKwargs() { + return rfn.hasKwargs(); + } + + /** Returns the location of the function's defining identifier. */ + @Override + public Location getLocation() { + return rfn.getLocation(); + } + + /** + * Returns the name of the function, or "lambda" if anonymous. Implicit functions (those not + * created by a def statement), may have names such as "" or "". + */ + @Override + public String getName() { + return rfn.getName(); + } + + /** + * Returns the value denoted by the function's doc string literal (trimmed if necessary), or null + * if absent. + */ + @Nullable + public String getDocumentation() { + String documentation = rfn.getDocumentation(); + return documentation != null ? Starlark.trimDocString(documentation) : null; + } + + public Module getModule() { + return module; + } + + @Override + public Object fastcall(StarlarkThread thread, Object[] positional, Object[] named) + throws EvalException, InterruptedException { + if (!thread.isRecursionAllowed() && thread.isRecursiveCall(this)) { + throw Starlark.errorf("function '%s' called recursively", getName()); + } + + // Compute the effective parameter values + // and update the corresponding variables. + StarlarkThread.Frame fr = thread.frame(0); + fr.locals = processArgs(thread.mutability(), positional, named); + + // Spill indicated locals to cells. + for (int index : rfn.getCellIndices()) { + fr.locals[index] = new Cell(fr.locals[index]); + } + + return Eval.execFunctionBody(fr, rfn.getBody()); + } + + Cell getFreeVar(int index) { + return (Cell) freevars.get(index); + } + + void export(StarlarkThread thread, String name) { + // Checks that thread is the one that defines the StarlarkFunction. It's possible for one + // StarlarkFunction to be exported in different places. + if (!token.getOwner().equals(thread.getOwner())) { + return; + } + if (token.isGlobal()) { + // Keeps only the first token if the same function is exported under multiple aliases. + return; + } + token = token.exportAs(name); + } + + @Override + public void repr(Printer printer) { + // TODO(adonovan): use the file name instead. But that's a breaking Bazel change. + Object clientData = module.getClientData(); + + printer.append(""); + } + + // Checks the positional and named arguments to ensure they match the signature. It returns a new + // array of effective parameter values corresponding to the parameters of the signature. The + // returned array has size of locals and is directly pushed to the stack. + // Newly allocated values (e.g. a **kwargs dict) use the Mutability mu. + // + // If the function has optional parameters, their default values are supplied by getDefaultValue. + private Object[] processArgs(Mutability mu, Object[] positional, Object[] named) + throws EvalException { + + // This is the general schema of a function: + // + // def f(p1, p2=dp2, p3=dp3, *args, k1, k2=dk2, k3, **kwargs) + // + // The p parameters are non-kwonly, and may be specified positionally. + // The k parameters are kwonly, and must be specified by name. + // The defaults tuple is (dp2, dp3, MANDATORY, dk2, MANDATORY). + // The missing prefix (p1) is assumed to be all MANDATORY. + // + // Arguments are processed as follows: + // - positional arguments are bound to a prefix of [p1, p2, p3]. + // - surplus positional arguments are bound to *args. + // - keyword arguments are bound to any of {p1, p2, p3, k1, k2, k3}; + // duplicate bindings are rejected. + // - surplus keyword arguments are bound to **kwargs. + // - defaults are bound to each parameter from p2 to k3 if no value was set. + // default values come from the tuple above. + // It is an error if the defaults tuple entry for an unset parameter is MANDATORY. + + ImmutableList names = rfn.getParameterNames(); + + Object[] locals = new Object[rfn.getLocals().size()]; + + // numOrdinaryParams is the number of ordinary (non-residual, non-kwonly) parameters. + int numOrdinaryParams = getNumOrdinaryParameters(); + + // nparams is the number of all non-residual parameters. + int nparams = numOrdinaryParams + getNumKeywordOnlyParameters(); + + // Too many positional args? + int positionalCount = positional.length; + if (positionalCount > numOrdinaryParams) { + if (!rfn.hasVarargs()) { + if (numOrdinaryParams > 0) { + throw Starlark.errorf( + "%s() accepts no more than %d positional argument%s but got %d", + getName(), numOrdinaryParams, plural(numOrdinaryParams), positionalCount); + } else { + throw Starlark.errorf( + "%s() does not accept positional arguments, but got %d", getName(), positionalCount); + } + } + positionalCount = numOrdinaryParams; + } + // Inv: n is number of positional arguments that are not surplus. + + // Bind positional arguments to non-kwonly parameters. + for (int i = 0; i < positionalCount; i++) { + locals[i] = positional[i]; + } + + // Bind surplus positional arguments to *args parameter. + if (rfn.hasVarargs()) { + locals[nparams] = + Tuple.wrap(Arrays.copyOfRange(positional, positionalCount, positional.length)); + } + + List unexpected = null; + + // Named arguments. + LinkedHashMap kwargs = null; + if (rfn.hasKwargs()) { + // To avoid Dict overhead, we populate a LinkedHashMap and then pass it to Dict.wrap() + // afterwards. (The contract of Dict.wrap prohibits us from modifying the map once the Dict is + // created.) + kwargs = Maps.newLinkedHashMapWithExpectedSize(1); + } + for (int i = 0; i < named.length; i += 2) { + String keyword = (String) named[i]; // safe + Object value = named[i + 1]; + int pos = names.indexOf(keyword); // the list should be short, so linear scan is OK. + if (0 <= pos && pos < nparams) { + // keyword is the name of a named parameter + if (locals[pos] != null) { + throw Starlark.errorf("%s() got multiple values for parameter '%s'", getName(), keyword); + } + locals[pos] = value; + + } else if (kwargs != null) { + // residual keyword argument + if (kwargs.put(keyword, value) != null) { + throw Starlark.errorf( + "%s() got multiple values for keyword argument '%s'", getName(), keyword); + } + + } else { + // unexpected keyword argument + if (unexpected == null) { + unexpected = new ArrayList<>(); + } + unexpected.add(keyword); + } + } + if (unexpected != null) { + // Give a spelling hint if there is exactly one. + // More than that suggests the wrong function was called. + throw Starlark.errorf( + "%s() got unexpected keyword argument%s: %s%s", + getName(), + plural(unexpected.size()), + Joiner.on(", ").join(unexpected), + unexpected.size() == 1 + ? SpellChecker.didYouMean(unexpected.get(0), names.subList(0, nparams)) + : ""); + } + if (kwargs != null) { + locals[rfn.getParameters().size() - 1] = Dict.wrap(mu, kwargs); + } + + // Apply defaults and report errors for missing required arguments. + int m = nparams - defaultValues.size(); // first default + List missingPositional = null; + List missingKwonly = null; + for (int i = positionalCount; i < nparams; i++) { + // provided? + if (locals[i] != null) { + continue; + } + + // optional? + if (i >= m) { + Object dflt = defaultValues.get(i - m); + if (dflt != MANDATORY) { + locals[i] = dflt; + continue; + } + } + + // missing + if (i < numOrdinaryParams) { + if (missingPositional == null) { + missingPositional = new ArrayList<>(); + } + missingPositional.add(names.get(i)); + } else { + if (missingKwonly == null) { + missingKwonly = new ArrayList<>(); + } + missingKwonly.add(names.get(i)); + } + } + if (missingPositional != null) { + throw Starlark.errorf( + "%s() missing %d required positional argument%s: %s", + getName(), + missingPositional.size(), + plural(missingPositional.size()), + Joiner.on(", ").join(missingPositional)); + } + if (missingKwonly != null) { + throw Starlark.errorf( + "%s() missing %d required keyword-only argument%s: %s", + getName(), + missingKwonly.size(), + plural(missingKwonly.size()), + Joiner.on(", ").join(missingKwonly)); + } + + return locals; + } + + private static String plural(int n) { + return n == 1 ? "" : "s"; + } + + @Override + public String toString() { + StringBuilder out = new StringBuilder(); + out.append(getName()); + out.append('('); + String sep = ""; + // TODO(adonovan): include *, ** tokens. + for (String param : getParameterNames()) { + out.append(sep).append(param); + sep = ", "; + } + out.append(')'); + return out.toString(); + } + + public SymbolGenerator.Symbol getToken() { + return token; + } + + @Override + public int hashCode() { + return token.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof StarlarkFunction)) { + return false; + } + return token.equals(((StarlarkFunction) obj).token); + } + + @Override + public boolean isImmutable() { + // Only correct because closures are not yet supported. + return true; + } + + // The MANDATORY sentinel indicates a slot in the defaultValues + // tuple corresponding to a required parameter. + // It is not visible to Java or Starlark code. + static final Object MANDATORY = new Mandatory(); + + private static class Mandatory implements StarlarkValue {} + + // A Cell is a local variable shared between an inner and an outer function. + // It is a StarlarkValue because it is a stack operand and a Tuple element, + // but it is not visible to Java or Starlark code. + static final class Cell implements StarlarkValue { + Object x; + + Cell(Object x) { + this.x = x; + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkIndexable.java b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkIndexable.java new file mode 100644 index 000000000..db44bef1d --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkIndexable.java @@ -0,0 +1,54 @@ +// Copyright 2015 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +/** + * A Starlark value that support indexed access ({@code object[key]}) and membership tests ({@code + * key in object}). + * + *

Implementations of this interface come in three flavors: map-like, sequence-like, and + * string-like. + * + *

    + *
  • For map-like objects, 'x in y' should return True when 'y[x]' is valid; otherwise, it + * should either be False or a failure. Examples: dict. + *
  • For sequence-like objects, 'x in y' should return True when 'x == y[i]' for some integer + * 'i'; otherwise, it should either be False or a failure. Examples: list, tuple, and string + * (which, notably, is not a {@link Sequence}). + *
  • For string-like objects, 'x in y' should return True when 'x' is a substring of 'y', i.e. + * 'x[i] == y[i + n]' for some 'n' and all i in [0, len(x)). Examples: string. + *
+ */ +public interface StarlarkIndexable extends StarlarkMembershipTestable { + + /** Returns the value associated with the given key. */ + Object getIndex(StarlarkSemantics semantics, Object key) throws EvalException; + + /** + * A variant of {@link StarlarkIndexable} that also provides a StarlarkThread instance on method + * calls. + */ + // TODO(brandjon): Consider replacing this subinterface by changing StarlarkIndexable's methods' + // signatures to take StarlarkThread in place of StarlarkSemantics. + interface Threaded { + /** {@see StarlarkIndexable.getIndex} */ + Object getIndex(StarlarkThread starlarkThread, StarlarkSemantics semantics, Object key) + throws EvalException; + + /** {@see StarlarkIndexable.containsKey} */ + boolean containsKey(StarlarkThread starlarkThread, StarlarkSemantics semantics, Object key) + throws EvalException; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkInt.java b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkInt.java new file mode 100644 index 000000000..10e9a0bb4 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkInt.java @@ -0,0 +1,798 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import java.math.BigInteger; +import net.starlark.java.annot.StarlarkBuiltin; + +/** The Starlark int data type. */ +@StarlarkBuiltin( + name = "int", + category = "core", + doc = + "The type of integers in Starlark. Starlark integers may be of any magnitude; arithmetic" + + " is exact. Examples of integer expressions:
" + + "
153\n"
+            + "0x2A  # hexadecimal literal\n"
+            + "0o54  # octal literal\n"
+            + "23 * 2 + 5\n"
+            + "100 / -7\n"
+            + "100 % -7  # -5 (unlike in some other languages)\n"
+            + "int(\"18\")\n"
+            + "
") +public abstract class StarlarkInt implements StarlarkValue, Comparable { + + // A cache of small integers >= LEAST_SMALLINT. + private static final int LEAST_SMALLINT = -128; + private static final Int32[] smallints = new Int32[100_000]; + + static final StarlarkInt ZERO = StarlarkInt.of(0); + private static final StarlarkInt ONE = StarlarkInt.of(1); + private static final StarlarkInt MINUS_ONE = StarlarkInt.of(-1); + + /** Only nested classes of {@code StarlarkInt} are allowed to inherit it. */ + private StarlarkInt() {} + + /** Returns the Starlark int value that represents x. */ + public static StarlarkInt of(int x) { + int index = x - LEAST_SMALLINT; // (may overflow) + if (0 <= index && index < smallints.length) { + Int32 xi = smallints[index]; + if (xi == null) { + xi = new Int32(x); + smallints[index] = xi; + } + return xi; + } + return new Int32(x); + } + + /** Returns the Starlark int value that represents x. */ + public static StarlarkInt of(long x) { + if ((long) (int) x == x) { + return StarlarkInt.of((int) x); + } + return new Int64(x); + } + + /** Returns the Starlark int value that represents x. */ + public static StarlarkInt of(BigInteger x) { + if (x.bitLength() < 64) { + return StarlarkInt.of(x.longValue()); + } + return new Big(x); + } + + /** + * Returns the StarlarkInt value that most closely approximates x. + * + * @throws IllegalArgumentException is x is not finite. + */ + static StarlarkInt ofFiniteDouble(double x) { + return StarlarkFloat.finiteDoubleToIntExact(x); + } + + /** + * Returns the int denoted by a literal string in the specified base, as if by the Starlark + * expression {@code int(s, base)}. + * + * @throws NumberFormatException if the input is invalid. + */ + public static StarlarkInt parse(String s, int base) { + String stringForErrors = s; + + if (s.isEmpty()) { + throw new NumberFormatException("empty string"); + } + + // +/- prefix? + boolean isNegative = false; + char c = s.charAt(0); + if (c == '+') { + s = s.substring(1); + } else if (c == '-') { + s = s.substring(1); + isNegative = true; + } + + String digits = s; + + // 0b 0o 0x prefix? + if (s.length() > 1 && s.charAt(0) == '0') { + int prefixBase = 0; + c = s.charAt(1); + if (c == 'b' || c == 'B') { + prefixBase = 2; + } else if (c == 'o' || c == 'O') { + prefixBase = 8; + } else if (c == 'x' || c == 'X') { + prefixBase = 16; + } + if (prefixBase != 0) { + if (base == 0 || base == prefixBase) { + base = prefixBase; + digits = s.substring(2); // strip prefix + } + } + } + + // No prefix, no base? Use decimal. + if (digits == s && base == 0) { + // Don't infer base when input starts with '0' due to octal/decimal ambiguity. + if (s.length() > 1 && s.charAt(0) == '0') { + throw new NumberFormatException( + "cannot infer base when string begins with a 0: " + Starlark.repr(stringForErrors)); + } + base = 10; + } + if (base < 2 || base > 36) { + throw new NumberFormatException( + String.format("invalid base %d (want 2 <= base <= 36)", base)); + } + + // Do not allow Long.parseLong and new BigInteger to accept another +/- sign. + if (digits.startsWith("+") || digits.startsWith("-")) { + throw new NumberFormatException( + String.format("invalid base-%d literal: %s", base, Starlark.repr(stringForErrors))); + } + + StarlarkInt result; + try { + result = StarlarkInt.of(Long.parseLong(digits, base)); + } catch (NumberFormatException unused1) { + try { + result = StarlarkInt.of(new BigInteger(digits, base)); + } catch (NumberFormatException unused2) { + throw new NumberFormatException( + String.format("invalid base-%d literal: %s", base, Starlark.repr(stringForErrors))); + } + } + return isNegative ? StarlarkInt.uminus(result) : result; + } + + // Subclass for values exactly representable in a Java int. + private static final class Int32 extends StarlarkInt { + final int v; + + Int32(int v) { + this.v = v; + } + + @Override + public int toInt(String what) { + return v; + } + + @Override + public long toLong(String what) { + return (long) v; + } + + @Override + protected long toLongFast() { + return (long) v; + } + + @Override + public BigInteger toBigInteger() { + return BigInteger.valueOf(v); + } + + @Override + public Number toNumber() { + return v; + } + + @Override + public int signum() { + return Integer.signum(v); + } + + @Override + public void repr(Printer printer) { + printer.append(v); + } + + @Override + public int hashCode() { + return 0x316c5239 * Integer.hashCode(v) ^ 0x67c4a7d5; + } + + @Override + public boolean equals(Object that) { + return (that instanceof Int32 && this.v == ((Int32) that).v) + || (that instanceof StarlarkFloat && intEqualsFloat(this, (StarlarkFloat) that)); + } + } + + // Subclass for values exactly representable in a Java long. + private static final class Int64 extends StarlarkInt { + final long v; + + Int64(long v) { + this.v = v; + } + + @Override + public long toLong(String what) { + return v; + } + + @Override + protected long toLongFast() { + return v; + } + + @Override + public BigInteger toBigInteger() { + return BigInteger.valueOf(v); + } + + @Override + public Number toNumber() { + return v; + } + + @Override + public int signum() { + return Long.signum(v); + } + + @Override + public void repr(Printer printer) { + printer.append(v); + } + + @Override + public int hashCode() { + return 0x67c4a7d5 * Long.hashCode(v) ^ 0xee914a1b; + } + + @Override + public boolean equals(Object that) { + return (that instanceof Int64 && this.v == ((Int64) that).v) + || (that instanceof StarlarkFloat && intEqualsFloat(this, (StarlarkFloat) that)); + } + } + + // Subclass for values not exactly representable in a long. + private static final class Big extends StarlarkInt { + final BigInteger v; + + Big(BigInteger v) { + this.v = v; + } + + @Override + public BigInteger toBigInteger() { + return v; + } + + @Override + public Number toNumber() { + return v; + } + + @Override + public int signum() { + return v.signum(); + } + + @Override + public void repr(Printer printer) { + printer.append(v.toString()); + } + + @Override + public int hashCode() { + return 0xee914a1b * v.hashCode() ^ 0x6406918f; + } + + @Override + public boolean equals(Object that) { + return (that instanceof Big && this.v.equals(((Big) that).v)) + || (that instanceof StarlarkFloat && intEqualsFloat(this, (StarlarkFloat) that)); + } + } + + /** Returns the value of this StarlarkInt as a Number (Integer, Long, or BigInteger). */ + public abstract Number toNumber(); + + /** Returns the signum of this StarlarkInt (-1, 0, or +1). */ + public abstract int signum(); + + /** Returns this StarlarkInt as a string of decimal digits. */ + @Override + public String toString() { + if (this instanceof Int32) { + return Integer.toString(((Int32) this).v); + } else if (this instanceof Int64) { + return Long.toString(((Int64) this).v); + } else { + return toBigInteger().toString(); + } + } + + @Override + public abstract void repr(Printer printer); + + /** Returns the signed int32 value of this StarlarkInt, or fails if not exactly representable. */ + public int toInt(String what) throws EvalException { + throw Starlark.errorf("got %s for %s, want value in signed 32-bit range", this, what); + } + + /** Returns the signed int64 value of this StarlarkInt, or fails if not exactly representable. */ + public long toLong(String what) throws EvalException { + throw Starlark.errorf("got %s for %s, want value in the signed 64-bit range", this, what); + } + + // A preallocated exception used to indicate overflow errors without the cost of allocation. + private static final Overflow OVERFLOW = new Overflow(); + + private static final class Overflow extends Exception {} + + /** + * Similar to {@link #toLong(String)}, but faster: exception is not allocated and stack trace is + * not collected. + */ + protected long toLongFast() throws Overflow { + throw OVERFLOW; + } + + /** Returns the nearest IEEE-754 double-precision value closest to this int, which may be ±Inf. */ + public double toDouble() { + if (this instanceof Int32) { + return ((Int32) this).v; + } else if (this instanceof Int64) { + return ((Int64) this).v; + } else { + return toBigInteger().doubleValue(); // may be ±Inf + } + } + + /** + * Returns the nearest IEEE-754 double-precision value closest to this int. + * + * @throws EvalException is the int is to large to represent as a finite float value. + */ + public double toFiniteDouble() throws EvalException { + double d = toDouble(); + if (!Double.isFinite(d)) { + throw Starlark.errorf("int too large to convert to float"); + } + return d; + } + + /** Returns the BigInteger value of this StarlarkInt. */ + public abstract BigInteger toBigInteger(); + + /** + * Returns the value of this StarlarkInt as a Java signed 32-bit int. + * + * @throws IllegalArgumentException if this int is not in that value range. + */ + public final int toIntUnchecked() throws IllegalArgumentException { + if (this instanceof Int32) { + return ((Int32) this).v; + } + // Use a constant exception to avoid allocation. + // This operator is provided for fast access and case discrimination. + // Use toInt(String) for user-visible errors. + throw NOT_INT32; + } + + private static final IllegalArgumentException NOT_INT32 = + new IllegalArgumentException("not a signed 32-bit value"); + + /** Returns the result of truncating this value into the signed 32-bit range. */ + public int truncateToInt() { + if (this instanceof Int32) { + return ((Int32) this).v; + } else if (this instanceof Int64) { + return (int) ((Int64) this).v; + } else { + return toBigInteger().intValue(); + } + } + + @Override + public boolean isImmutable() { + return true; + } + + @Override + public boolean truth() { + return this != ZERO; + } + + @Override + public int compareTo(StarlarkInt x) { + return compare(this, x); + } + + // binary operators + + /** Returns signum(x - y). */ + public static int compare(StarlarkInt x, StarlarkInt y) { + // If both arguments are big, we compare BigIntegers. + // If neither argument is big, we compare longs. + // If only one argument is big, its magnitude is greater + // than the other operand, so only its sign matters. + // + // We avoid unnecessary branches. + try { + long xl = x.toLongFast(); + try { + long yl = y.toLongFast(); + return Long.compare(xl, yl); // (long, long) + } catch (Overflow unused) { + return -((Big) y).v.signum(); // (long, big) + } + } catch (Overflow unused) { + return y instanceof Big + ? ((Big) x).v.compareTo(((Big) y).v) // (big, big) + : ((Big) x).v.signum(); // (big, long) + } + } + + /** Returns x + y. */ + public static StarlarkInt add(StarlarkInt x, StarlarkInt y) { + if (x instanceof Int32 && y instanceof Int32) { + long xl = ((Int32) x).v; + long yl = ((Int32) y).v; + return StarlarkInt.of(xl + yl); + } + + // We avoid Math.addExact and its overheads of exception allocation. + try { + long xl = x.toLongFast(); + long yl = y.toLongFast(); + long zl = xl + yl; + boolean overflow = ((xl ^ zl) & (yl ^ zl)) < 0; // see Hacker's Delight, chapter 2 + if (!overflow) { + return StarlarkInt.of(zl); + } + } catch (Overflow unused) { + /* fall through */ + } + + BigInteger xbig = x.toBigInteger(); + BigInteger ybig = y.toBigInteger(); + BigInteger zbig = xbig.add(ybig); + return StarlarkInt.of(zbig); + } + + /** Returns x - y. */ + public static StarlarkInt subtract(StarlarkInt x, StarlarkInt y) { + if (x instanceof Int32 && y instanceof Int32) { + long xl = ((Int32) x).v; + long yl = ((Int32) y).v; + return StarlarkInt.of(xl - yl); + } + + // We avoid Math.subtractExact and its overhead of exception allocation. + try { + long xl = x.toLongFast(); + long yl = y.toLongFast(); + long zl = xl - yl; + boolean overflow = ((xl ^ yl) & (xl ^ zl)) < 0; // see Hacker's Delight, chapter 2 + if (!overflow) { + return StarlarkInt.of(zl); + } + } catch (Overflow unused) { + /* fall through */ + } + + BigInteger xbig = x.toBigInteger(); + BigInteger ybig = y.toBigInteger(); + BigInteger zbig = xbig.subtract(ybig); + return StarlarkInt.of(zbig); + } + + /** Returns x * y. */ + public static StarlarkInt multiply(StarlarkInt x, StarlarkInt y) { + // Fast path for common case: int32 * int32. + if (x instanceof Int32 && y instanceof Int32) { + long xl = ((Int32) x).v; + long yl = ((Int32) y).v; + return StarlarkInt.of(xl * yl); + } + + try { + long xl = x.toLongFast(); + long yl = y.toLongFast(); + + // Signed int128 multiplication, using Hacker's Delight 8-2 + // (High-Order Half of 64-Bit Product) extended to 128 bits. + // TODO(adonovan): use Math.multiplyHigh when Java 9 becomes available. + long xlo = xl & 0xFFFFFFFFL; + long xhi = xl >> 32; + long ylo = yl & 0xFFFFFFFFL; + long yhi = yl >> 32; + long zlo = xlo * ylo; + long t = xhi * ylo + (zlo >>> 32); + long z1 = t & 0xFFFFFFFFL; + long z2 = t >> 32; + z1 += xlo * yhi; + + // high and low arms of result + long z128hi = xhi * yhi + z2 + (z1 >> 32); + long z128lo = xl * yl; + + // Check int128 result is within int64 range. + if (z128hi == (z128lo & Long.MIN_VALUE) >> 63) { + return StarlarkInt.of(z128lo); + } + + /* overflow */ + + } catch (Overflow unused) { + /* fall through */ + } + + // Avoid unnecessary conversion to BigInteger if the other operand is -1, 0, 1. + // (Also makes self-test below faster.) + if (x == ZERO || y == ONE) { + return x; + } else if (y == ZERO || x == ONE) { + return y; + } else if (x == MINUS_ONE) { + return StarlarkInt.uminus(y); + } else if (y == MINUS_ONE) { + return StarlarkInt.uminus(x); + } + + BigInteger xbig = x.toBigInteger(); + BigInteger ybig = y.toBigInteger(); + BigInteger zbig = xbig.multiply(ybig); + StarlarkInt z = StarlarkInt.of(zbig); + // cheap self-test + if (!(z instanceof Big)) { + throw new AssertionError( + String.format( + "bug in multiplication: %s * %s = %s, must be long multiplication", x, y, z)); + } + return z; + } + + /** Returns x // y (floor of integer division). */ + public static StarlarkInt floordiv(StarlarkInt x, StarlarkInt y) throws EvalException { + if (y == ZERO) { + throw Starlark.errorf("integer division by zero"); + } + try { + long xl = x.toLongFast(); + long yl = y.toLongFast(); + // http://python-history.blogspot.com/2010/08/why-pythons-integer-division-floors.html + if (xl == Long.MIN_VALUE && yl == -1) { + /* sole case in which quotient doesn't fit in long */ + } else { + long quo = Math.floorDiv(xl, yl); + return StarlarkInt.of(quo); + } + /* overflow */ + } catch (Overflow unused) { + /* fall through */ + } + + BigInteger xbig = x.toBigInteger(); + BigInteger ybig = y.toBigInteger(); + BigInteger[] quorem = xbig.divideAndRemainder(ybig); + if ((xbig.signum() < 0) != (ybig.signum() < 0) && quorem[1].signum() != 0) { + quorem[0] = quorem[0].subtract(BigInteger.ONE); + } + return StarlarkInt.of(quorem[0]); + } + + /** Returns x % y. */ + public static StarlarkInt mod(StarlarkInt x, StarlarkInt y) throws EvalException { + if (y == ZERO) { + throw Starlark.errorf("integer modulo by zero"); + } + try { + long xl = x.toLongFast(); + long yl = y.toLongFast(); + // In Starlark, the sign of the result is the sign of the divisor. + return StarlarkInt.of(Math.floorMod(xl, yl)); + } catch (Overflow unused) { + /* fall through */ + } + + BigInteger xbig = x.toBigInteger(); + BigInteger ybig = y.toBigInteger(); + BigInteger zbig = xbig.remainder(ybig); + if ((x.signum() < 0) != (y.signum() < 0) && zbig.signum() != 0) { + zbig = zbig.add(ybig); + } + return StarlarkInt.of(zbig); + } + + /** Returns x >> y. */ + public static StarlarkInt shiftRight(StarlarkInt x, StarlarkInt y) throws EvalException { + int yi = y.toInt("shift count"); + if (yi < 0) { + throw Starlark.errorf("negative shift count: %d", yi); + } + try { + long xl = x.toLongFast(); + if (yi >= Long.SIZE) { + return xl < 0 ? StarlarkInt.of(-1) : ZERO; + } + return StarlarkInt.of(xl >> yi); + } catch (Overflow unused) { + /* fall through */ + } + + BigInteger xbig = x.toBigInteger(); + BigInteger zbig = xbig.shiftRight(yi); + return StarlarkInt.of(zbig); + } + + /** Returns x << y. */ + public static StarlarkInt shiftLeft(StarlarkInt x, StarlarkInt y) throws EvalException { + int yi = y.toInt("shift count"); + if (yi < 0) { + throw Starlark.errorf("negative shift count: %d", yi); + } else if (yi >= 512) { + throw Starlark.errorf("shift count too large: %d", yi); + } + try { + long xl = x.toLongFast(); + long z = xl << yi; // only uses low 6 bits of yi + if ((z >> yi) == xl && yi < 64) { + return StarlarkInt.of(z); + } + /* overflow */ + } catch (Overflow unused) { + /* fall through */ + } + + BigInteger xbig = x.toBigInteger(); + BigInteger zbig = xbig.shiftLeft(yi); + return StarlarkInt.of(zbig); + } + + /** Returns x ^ y. */ + public static StarlarkInt xor(StarlarkInt x, StarlarkInt y) { + try { + long xl = x.toLongFast(); + long yl = y.toLongFast(); + return StarlarkInt.of(xl ^ yl); + } catch (Overflow unused) { + /* fall through */ + } + + BigInteger xbig = x.toBigInteger(); + BigInteger ybig = y.toBigInteger(); + BigInteger zbig = xbig.xor(ybig); + return StarlarkInt.of(zbig); + } + + /** Returns x | y. */ + public static StarlarkInt or(StarlarkInt x, StarlarkInt y) { + try { + long xl = x.toLongFast(); + long yl = y.toLongFast(); + return StarlarkInt.of(xl | yl); + } catch (Overflow unused) { + /* fall through */ + } + + BigInteger xbig = x.toBigInteger(); + BigInteger ybig = y.toBigInteger(); + BigInteger zbig = xbig.or(ybig); + return StarlarkInt.of(zbig); + } + + /** Returns x & y. */ + public static StarlarkInt and(StarlarkInt x, StarlarkInt y) { + try { + long xl = x.toLongFast(); + long yl = y.toLongFast(); + return StarlarkInt.of(xl & yl); + } catch (Overflow unused) { + /* fall through */ + } + + BigInteger xbig = x.toBigInteger(); + BigInteger ybig = y.toBigInteger(); + BigInteger zbig = xbig.and(ybig); + return StarlarkInt.of(zbig); + } + + /** Returns ~x. */ + public static StarlarkInt bitnot(StarlarkInt x) { + try { + long xl = x.toLongFast(); + return StarlarkInt.of(~xl); + } catch (Overflow unused) { + /* fall through */ + } + + BigInteger xbig = ((Big) x).v; + return StarlarkInt.of(xbig.not()); + } + + /** Returns -x. */ + public static StarlarkInt uminus(StarlarkInt x) { + if (x instanceof Int32) { + long xl = ((Int32) x).v; + return StarlarkInt.of(-xl); + } + + if (x instanceof Int64) { + long xl = ((Int64) x).v; + if (xl != Long.MIN_VALUE) { + return StarlarkInt.of(-xl); + } + } + + BigInteger xbig = x.toBigInteger(); + BigInteger ybig = xbig.negate(); + return StarlarkInt.of(ybig); + } + + /** Reports whether int x exactly equals float y. */ + static boolean intEqualsFloat(StarlarkInt x, StarlarkFloat y) { + double yf = y.toDouble(); + return !Double.isNaN(yf) && compareIntAndDouble(x, yf) == 0; + } + + /** Returns an exact three-valued comparison of int x with (non-NaN) double y. */ + static int compareIntAndDouble(StarlarkInt x, double y) { + if (Double.isInfinite(y)) { + return y > 0 ? -1 : +1; + } + + // For Int32 and some Int64s, the toDouble conversion is exact. + if (x instanceof StarlarkInt.Int32 + || (x instanceof StarlarkInt.Int64 && longHasExactDouble(((Int64) x).v))) { + // Avoid Double.compare: it believes -0.0 < 0.0. + double xf = x.toDouble(); + if (xf > y) { + return +1; + } else if (xf < y) { + return -1; + } + return 0; + } + + // If signs differ, we needn't look at magnitude. + int xsign = x.signum(); + int ysign = (int) Math.signum(y); + if (xsign > ysign) { + return +1; + } else if (xsign < ysign) { + return -1; + } + + // Left-shift either the int or the float mantissa, + // then compare the resulting integers. + int shift = StarlarkFloat.getShift(y); + BigInteger xbig = x.toBigInteger(); + if (shift < 0) { + xbig = xbig.shiftLeft(-shift); + } + BigInteger ybig = BigInteger.valueOf(StarlarkFloat.getMantissa(y)); + if (shift > 0) { + ybig = ybig.shiftLeft(shift); + } + return xbig.compareTo(ybig); + } + + private static boolean longHasExactDouble(long x) { + return (long) (double) x == x; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkIterable.java b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkIterable.java new file mode 100644 index 000000000..bcdeff2af --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkIterable.java @@ -0,0 +1,24 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +/** + * A StarlarkIterable value may be iterated by Starlark language constructs such as {@code for} + * loops, list and dict comprehensions, and {@code f(*args)}. + * + *

Functionally this interface is equivalent to {@code java.lang.Iterable}, but it additionally + * affirms that the iterability of a Java class should be exposed to Starlark programs. + */ +public interface StarlarkIterable extends StarlarkValue, Iterable {} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkList.java b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkList.java new file mode 100644 index 000000000..a8b30e10e --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkList.java @@ -0,0 +1,539 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import java.io.Serializable; +import java.util.AbstractCollection; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; +import java.util.NoSuchElementException; +import java.util.function.Supplier; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import net.starlark.java.annot.Param; +import net.starlark.java.annot.ParamType; +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; + +/** + * A StarlarkList is a mutable finite sequence of values. + * + *

Starlark operations on lists, including element update and the {@code append}, {@code insert}, + * and {@code extend} methods, may insert arbitrary Starlark values as list elements, regardless of + * the type argument used to reference to the list from Java code. Therefore, as long as a list is + * mutable, Java code should refer to it only through a type such as {@code StarlarkList} or + * {@code StarlarkList} to avoid undermining the type-safety of the Java application. Once the + * list becomes frozen, it is safe to {@link #cast} it to a more specific type that accurately + * reflects its elements, such as {@code StarlarkList}. + * + *

The following List methods, by inheriting their implementations from AbstractList, are + * effectively disabled. Use the corresponding methods with "element" in their name; they may report + * mutation failure by throwing a checked exception. + * + * 

+ *   boolean add(E)                    -- use addElement
+ *   boolean remove(Object)            -- use removeElement
+ *   boolean addAll(Collection)        -- use addElements
+ *   boolean addAll(int, Collection)
+ *   boolean removeAll(Collection)     -- use removeElements
+ *   boolean retainAll(Collection)
+ *   void clear()                      -- use clearElements
+ *   E set(int, E)                     -- use setElementAt
+ *   void add(int, E)                  -- use addElementAt
+ *   E remove(int)                     -- use removeElementAt
+ *
+ */ +@StarlarkBuiltin( + name = "list", + category = "core", + doc = + "The built-in list type. Example list expressions:
" + + "
x = [1, 2, 3]
" + + "Accessing elements is possible using indexing (starts from 0):
" + + "
e = x[1]   # e == 2
" + + "Lists support the + operator to concatenate two lists. Example:
" + + "
x = [1, 2] + [3, 4]   # x == [1, 2, 3, 4]\n"
+            + "x = [\"a\", \"b\"]\n"
+            + "x += [\"c\"]            # x == [\"a\", \"b\", \"c\"]
" + + "Similar to strings, lists support slice operations:" + + "
['a', 'b', 'c', 'd'][1:3]   # ['b', 'c']\n"
+            + "['a', 'b', 'c', 'd'][::2]  # ['a', 'c']\n"
+            + "['a', 'b', 'c', 'd'][3:0:-1]  # ['d', 'c', 'b']
" + + "Lists are mutable, as in Python.") +public abstract class StarlarkList extends AbstractCollection + implements Sequence, StarlarkValue, Mutability.Freezable, Comparable> { + + // It's always possible to overeat in small bites but we'll + // try to stop someone swallowing the world in one gulp. + static final int MAX_ALLOC = 1 << 30; + + static final Object[] EMPTY_ARRAY = {}; + + // Prohibit instantiation outside of package. + StarlarkList() {} + + /** + * Takes ownership of the supplied array of class Object[].class, and returns a new StarlarkList + * instance that initially wraps the array. The caller must not subsequently modify the array, but + * the StarlarkList instance may do so. + */ + static StarlarkList wrap(@Nullable Mutability mutability, Object[] elems) { + if (mutability == null || mutability.isFrozen()) { + switch (elems.length) { + case 0: + return empty(); + case 1: + return new ImmutableSingletonStarlarkList<>(elems[0]); + default: + return new RegularImmutableStarlarkList<>(elems); + } + } + return new MutableStarlarkList<>(mutability, elems, elems.length); + } + + @Override + public void checkHashable() throws EvalException { + // Even a frozen list is unhashable. + throw Starlark.errorf("unhashable type: 'list'"); + } + + /** Returns an empty frozen list of the desired type. */ + @SuppressWarnings("unchecked") + public static StarlarkList empty() { + return (StarlarkList) RegularImmutableStarlarkList.EMPTY; + } + + /** Returns a new, empty list with the specified Mutability. */ + public static StarlarkList newList(Mutability mutability) { + return wrap(mutability, EMPTY_ARRAY); + } + + /** + * Returns a {@code StarlarkList} whose items are given by an iterable and which has the given + * {@link Mutability}. If {@code mutability} is null, the list is immutable. + */ + public static StarlarkList copyOf( + @Nullable Mutability mutability, Iterable elems) { + if (mutability == null + && elems instanceof StarlarkList + && ((StarlarkList) elems).isImmutable()) { + @SuppressWarnings("unchecked") + StarlarkList list = (StarlarkList) elems; // safe + return list; + } + + Object[] array = Iterables.toArray(elems, Object.class); + checkElemsValid(array); + return wrap(mutability, array); + } + + private static void checkElemsValid(Object[] elems) { + for (Object elem : elems) { + Starlark.checkValid(elem); + } + } + + /** + * Returns an immutable list with the given elements. Equivalent to {@code copyOf(null, elems)}. + */ + public static StarlarkList immutableCopyOf(Iterable elems) { + return copyOf(null, elems); + } + + /** + * Creates an immutable {@link StarlarkList} with lazily supplied elements. + * + *

The given supplier is not invoked until the list is accessed and is invoked at most once. + * This can be used to create a {@link StarlarkList} while deferring an expensive computation + * until the list is actually accessed. + */ + public static StarlarkList lazyImmutable(SerializableListSupplier supplier) { + return new LazyImmutableStarlarkList<>(supplier); + } + + /** An associated convenience type for LazyImmutableStarlarkLists */ + public interface SerializableListSupplier extends Supplier>, Serializable {} + + /** + * Returns a {@code StarlarkList} with the given items and the {@link Mutability}. If {@code + * mutability} is null, the list is immutable. + */ + public static StarlarkList of(@Nullable Mutability mutability, T... elems) { + if (elems.length == 0) { + return newList(mutability); + } + + checkElemsValid(elems); + return wrap(mutability, Arrays.copyOf(elems, elems.length, Object[].class)); + } + + /** Returns an immutable {@code StarlarkList} with the given items. */ + public static StarlarkList immutableOf(T... elems) { + checkElemsValid(elems); + return wrap(null, Arrays.copyOf(elems, elems.length, Object[].class)); + } + + abstract Object[] elems(); + + /** + * Returns a new {@code StarlarkList} that is the concatenation of two {@code StarlarkList}s. The + * new list will have the given {@link Mutability}. + * + * @throws EvalException if the resulting list would be too large + */ + public static StarlarkList concat( + StarlarkList x, StarlarkList y, Mutability mutability) + throws EvalException { + int xsize = x.size(); + int ysize = y.size(); + Object[] res = new Object[addSizesAndFailIfExcessive(xsize, ysize)]; + System.arraycopy(x.elems(), 0, res, 0, xsize); + System.arraycopy(y.elems(), 0, res, xsize, ysize); + return wrap(mutability, res); + } + + protected static int addSizesAndFailIfExcessive(int xsize, int ysize) throws EvalException { + int sum = xsize + ysize; + if (sum < 0 || sum > MAX_ALLOC) { + throw Starlark.errorf("excessive capacity requested (%d + %d elements)", xsize, ysize); + } + return sum; + } + + @Nonnull + @Override + public Iterator iterator() { + return new Itr(); + } + + @Override + public int compareTo(StarlarkList that) { + return Sequence.compare(this, that); + } + + @Override + public boolean equals(Object that) { + // This slightly violates the java.util.List equivalence contract + // because it considers the class, not just the elements. + // This is needed because in Starlark lists are never equal to tuples, however in Java they both + // implement List interface. + return this == that + || (that instanceof StarlarkList && Sequence.sameElems(this, ((StarlarkList) that))); + } + + @Override + public int hashCode() { + // Hash the elements elems[0:size]. + int result = 1; + int size = size(); + Object[] elems = elems(); + for (int i = 0; i < size; i++) { + result = 31 * result + elems[i].hashCode(); + } + return 6047 + 4673 * result; + } + + @Override + public void repr(Printer printer) { + printer.printList(this, "[", ", ", "]"); + } + + // TODO(adonovan): StarlarkValue has 3 String methods yet still we need this fourth. Why? + @Override + public String toString() { + return Starlark.repr(this); + } + + /** Returns a new StarlarkList containing n consecutive repeats of this tuple. */ + public StarlarkList repeat(StarlarkInt n, Mutability mutability) throws EvalException { + if (n.signum() <= 0) { + return wrap(mutability, EMPTY_ARRAY); + } + + int ni = n.toInt("repeat"); + int size = size(); + long sz = (long) ni * size; + if (sz > MAX_ALLOC) { + throw Starlark.errorf("excessive repeat (%d * %d elements)", size, ni); + } + Object[] res = new Object[(int) sz]; + for (int i = 0; i < ni; i++) { + System.arraycopy(elems(), 0, res, i * size, size); + } + return wrap(mutability, res); + } + + @Override + public StarlarkList getSlice(Mutability mu, int start, int stop, int step) + throws EvalException { + RangeList indices = new RangeList(start, stop, step); + int n = indices.size(); + Object[] res = new Object[n]; + if (step == 1) { // common case + System.arraycopy(elems(), indices.at(0), res, 0, n); + } else { + Object[] elems = elems(); + for (int i = 0; i < n; ++i) { + res[i] = elems[indices.at(i)]; + } + } + return wrap(mu, res); + } + + /** + * Appends an element to the end of the list, after validating that mutation is allowed. + * + * @param element the element to add + */ + public abstract void addElement(E element) throws EvalException; + + /** + * Inserts an element at a given position to the list. + * + * @param index the new element's index + * @param element the element to add + */ + public abstract void addElementAt(int index, E element) throws EvalException; + + /** + * Appends all the elements to the end of the list. + * + * @param elements the elements to add + */ + public abstract void addElements(Iterable elements) throws EvalException; + + /** + * Removes the element at a given index. The index must already have been validated to be in + * range. + * + * @param index the index of the element to remove + */ + public abstract void removeElementAt(int index) throws EvalException; + + /** + * Sets the position at the given index to contain the given value. Precondition: {@code 0 <= + * index < size()}. + */ + public abstract void setElementAt(int index, E value) throws EvalException; + + @StarlarkMethod( + name = "remove", + doc = + "Removes the first item from the list whose value is x. " + + "It is an error if there is no such item.", + parameters = {@Param(name = "x", doc = "The object to remove.")}) + public void removeElement(Object x) throws EvalException { + int size = size(); + Object[] elems = elems(); + for (int i = 0; i < size; i++) { + if (elems[i].equals(x)) { + removeElementAt(i); + return; + } + } + throw Starlark.errorf("item %s not found in list", Starlark.repr(x)); + } + + @StarlarkMethod( + name = "append", + doc = "Adds an item to the end of the list.", + parameters = {@Param(name = "item", doc = "Item to add at the end.")}) + @SuppressWarnings("unchecked") + public void append(Object item) throws EvalException { + addElement((E) item); // unchecked + } + + @StarlarkMethod(name = "clear", doc = "Removes all the elements of the list.") + public abstract void clearElements() throws EvalException; + + @StarlarkMethod( + name = "insert", + doc = "Inserts an item at a given position.", + parameters = { + @Param(name = "index", doc = "The index of the given position."), + @Param(name = "item", doc = "The item.") + }) + @SuppressWarnings("unchecked") + public void insert(StarlarkInt index, Object item) throws EvalException { + addElementAt(EvalUtils.toIndex(index.toInt("index"), size()), (E) item); // unchecked + } + + @StarlarkMethod( + name = "extend", + doc = "Adds all items to the end of the list.", + parameters = {@Param(name = "items", doc = "Items to add at the end.")}) + public void extend(Object items) throws EvalException { + @SuppressWarnings("unchecked") + Iterable src = (Iterable) Starlark.toIterable(items); + addElements(src); + } + + @StarlarkMethod( + name = "index", + doc = + "Returns the index in the list of the first item whose value is x. " + + "It is an error if there is no such item.", + parameters = { + @Param(name = "x", doc = "The object to search."), + @Param( + name = "start", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), // TODO(adonovan): this is wrong + }, + defaultValue = "None", + named = true, // TODO(adonovan): this is wrong + doc = "The start index of the list portion to inspect."), + @Param( + name = "end", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), // TODO(adonovan): this is wrong + }, + defaultValue = "None", + named = true, // TODO(adonovan): this is wrong + doc = "The end index of the list portion to inspect.") + }) + public int index(Object x, Object start, Object end) throws EvalException { + int size = size(); + Object[] elems = elems(); + int i = start == Starlark.NONE ? 0 : EvalUtils.toIndex(Starlark.toInt(start, "start"), size); + int j = end == Starlark.NONE ? size : EvalUtils.toIndex(Starlark.toInt(end, "end"), size); + for (; i < j; i++) { + if (elems[i].equals(x)) { + return i; + } + } + throw Starlark.errorf("item %s not found in list", Starlark.repr(x)); + } + + @StarlarkMethod( + name = "pop", + doc = + "Removes the item at the given position in the list, and returns it. " + + "If no index is specified, " + + "it removes and returns the last item in the list.", + parameters = { + @Param( + name = "i", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), // TODO(adonovan): this is not what Python3 does + }, + defaultValue = "-1", + doc = "The index of the item.") + }) + public Object pop(Object i) throws EvalException { + int size = size(); + Object[] elems = elems(); + int arg = i == Starlark.NONE ? -1 : Starlark.toInt(i, "i"); + int index = EvalUtils.getSequenceIndex(arg, size); + Object result = elems[index]; + removeElementAt(index); + return result; + } + + /** + * Mutates this list in-place to reduce memory usage, and returns an optimized list (which might + * be the same as this instance). + * + *

This operation is not protected by the mutability mechanism. It is the caller's + * responsibility to ensure this list is not concurrently accessed during this method's execution. + * + *

The mutated list and the returned list are both equivalent to the original list. + * + *

The mutability must be frozen prior to calling this method. + */ + public StarlarkList unsafeOptimizeMemoryLayout() { + return this; + } + + private class Itr implements Iterator { + private int cursor = 0; + + @Override + public boolean hasNext() { + return cursor != size(); + } + + @Override + public E next() { + try { + int i = cursor; + E next = get(i); + cursor = i + 1; + return next; + } catch (IndexOutOfBoundsException e) { + throw new NoSuchElementException(e.getMessage()); + } + } + } + + // the following List methods are deliberately left unsupported for now, but could be implemented + // if the need ever arises + + @Override + @Nonnull + public List subList(int fromIndex, int toIndex) { + throw new UnsupportedOperationException(); + } + + @Override + @Nonnull + public ListIterator listIterator() { + throw new UnsupportedOperationException(); + } + + @Override + @Nonnull + public ListIterator listIterator(int index) { + throw new UnsupportedOperationException(); + } + + @Override + public int lastIndexOf(Object o) { + throw new UnsupportedOperationException(); + } + + @Override + public int indexOf(Object o) { + throw new UnsupportedOperationException(); + } + + @Override + public E set(int index, E element) { + throw new UnsupportedOperationException(); + } + + @Override + public void add(int index, E element) { + throw new UnsupportedOperationException(); + } + + @Override + public E remove(int index) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean addAll(int index, @Nonnull Collection c) { + throw new UnsupportedOperationException(); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkMembershipTestable.java b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkMembershipTestable.java new file mode 100644 index 000000000..747b0d42b --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkMembershipTestable.java @@ -0,0 +1,24 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +/** + * A Starlark value that support membership tests ({@code key in object} and {@code key not in + * object}). + */ +public interface StarlarkMembershipTestable extends StarlarkValue { + /** Returns whether the key is in the object. */ + boolean containsKey(StarlarkSemantics semantics, Object key) throws EvalException; +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkSemantics.java b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkSemantics.java new file mode 100644 index 000000000..658977e01 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkSemantics.java @@ -0,0 +1,262 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package net.starlark.java.eval; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.util.Map; +import java.util.TreeMap; + +/** + * A StarlarkSemantics is an immutable set of optional name/value pairs that affect the dynamic + * behavior of Starlark operators and built-in functions, both core and application-defined. + * + *

For extensibility, a StarlarkSemantics only records a name/value pair when the value differs + * from the default value appropriate to that name. Values of most types are accessed using a {@link + * Key}, which defines the name, type, and default value for an entry. Boolean values are accessed + * using a string key; the string must have the prefix "+" or "-", indicating the default value: + + * for true, - for false. The reason for the special treatment of boolean entries is that they may + * enable or disable methods and parameters in the StarlarkMethod annotation system, and it is not + * possible to refer to a Key from a Java annotation, only a string. + * + *

It is the client's responsibility to ensure that a StarlarkSemantics does not encounter + * multiple Keys of the same name but different value types. + * + *

For Bazel's semantics options, see {@link packages.semantics.BuildLanguageOptions}. + * + *

For options that affect the static behavior of the Starlark frontend (lexer, parser, + * validator, compiler), see {@link FileOptions}. + */ +public class StarlarkSemantics { + + /** + * Returns the empty semantics, in which every option has its default value. + * + *

Usage note: Usually all Starlark evaluation contexts (i.e., {@link StarlarkThread}s + * or other interpreter APIs that accept a {@code StarlarkSemantics}) within the same application + * should use the same semantics. Otherwise, different pieces of code -- or even the same code + * when executed in different capacities -- could produce diverging results. It is therefore + * generally a code smell to use the {@code DEFAULT} semantics rather than propagating a {@code + * StarlarkSemantics} from another context. + */ + public static final StarlarkSemantics DEFAULT = new StarlarkSemantics(ImmutableMap.of()); + + // A map entry must be accessed by Key iff its name has no [+-] prefix. + // Key is permitted too. + // The map keys are sorted but we avoid ImmutableSortedMap due to observed inefficiency. + private final ImmutableMap map; + private final int hashCode; + + private StarlarkSemantics(ImmutableMap map) { + this.map = map; + this.hashCode = map.hashCode(); + } + + protected StarlarkSemantics(StarlarkSemantics otherSemantics) { + this(otherSemantics.map); + } + + /** Returns the value of a boolean option, which must have a [+-] prefix. */ + public final boolean getBool(String name) { + char prefix = name.charAt(0); + Preconditions.checkArgument(prefix == '+' || prefix == '-'); + boolean defaultValue = prefix == '+'; + Boolean v = (Boolean) map.get(name); // prefix => cast cannot fail + return v != null ? v : defaultValue; + } + + /** Returns the value of the option denoted by {@code key}. */ + public final T get(Key key) { + @SuppressWarnings("unchecked") // safe, if Key.names are unique + T v = (T) map.get(key.name); + return v != null ? v : key.defaultValue; + } + + // TODO(bazel-team): This exists solely for BuiltinsInternalModule#getFlag, which allows a + // (privileged) Starlark caller to programmatically retrieve a flag's value without knowing its + // schema and default value. Reconsider whether we should support that use case from this class. + /** + * Returns the value of the option with the given name, or the default value if it is not set or + * does not exist. + */ + public final Object getGeneric(String name, Object defaultValue) { + Object v = map.get(name); + // Try boolean prefixes if that didn't work. + if (v == null) { + v = map.get("+" + name); + } + if (v == null) { + v = map.get("-" + name); + } + return v != null ? v : defaultValue; + } + + /** A Key identifies an option, providing its name, type, and default value. */ + public static final class Key { + public final String name; + public final T defaultValue; + + /** + * Constructs a key. The name must not start with [+-]. The value must not be subsequently + * modified. + */ + public Key(String name, T defaultValue) { + char prefix = name.charAt(0); + Preconditions.checkArgument(prefix != '-' && prefix != '+'); + this.name = name; + this.defaultValue = Preconditions.checkNotNull(defaultValue); + } + + @Override + public String toString() { + return this.name; + } + } + + /** + * Returns a new builder that initially holds the same key/value pairs as this StarlarkSemantics. + */ + public final Builder toBuilder() { + return new Builder(new TreeMap<>(map)); + } + + /** Returns a new empty builder. */ + public static Builder builder() { + return new Builder(new TreeMap<>()); + } + + /** A Builder is a mutable container used to construct an immutable StarlarkSemantics. */ + public static final class Builder { + private final TreeMap map; + + private Builder(TreeMap map) { + this.map = map; + } + + /** Sets the value for the specified key. */ + @CanIgnoreReturnValue + public Builder set(Key key, T value) { + if (!value.equals(key.defaultValue)) { + map.put(key.name, value); + } else { + map.remove(key.name); + } + return this; + } + + /** Sets the value for the boolean key, which must have a [+-] prefix. */ + @CanIgnoreReturnValue + public Builder setBool(String name, boolean value) { + char prefix = name.charAt(0); + Preconditions.checkArgument(prefix == '+' || prefix == '-'); + boolean defaultValue = prefix == '+'; + if (value != defaultValue) { + map.put(name, value); + } else { + map.remove(name); + } + return this; + } + + /** Returns an immutable StarlarkSemantics. */ + public StarlarkSemantics build() { + return new StarlarkSemantics(ImmutableMap.copyOf(map)); + } + } + + /** + * Returns true if a feature attached to the given toggling flags should be enabled. + * + *

    + *
  • If both parameters are empty, this indicates the feature is not controlled by flags, and + * should thus be enabled. + *
  • If the {@code enablingFlag} parameter is non-empty, this returns true if and only if that + * flag is true. (This represents a feature that is only on if a given flag is *on*). + *
  • If the {@code disablingFlag} parameter is non-empty, this returns true if and only if + * that flag is false. (This represents a feature that is only on if a given flag is *off*). + *
  • It is illegal to pass both parameters as non-empty. + *
+ */ + final boolean isFeatureEnabledBasedOnTogglingFlags(String enablingFlag, String disablingFlag) { + Preconditions.checkArgument( + enablingFlag.isEmpty() || disablingFlag.isEmpty(), + "at least one of 'enablingFlag' or 'disablingFlag' must be empty"); + if (!enablingFlag.isEmpty()) { + return this.getBool(enablingFlag); + } else if (!disablingFlag.isEmpty()) { + return !this.getBool(disablingFlag); + } else { + return true; + } + } + + /** + * Returns a possibly different {@link StarlarkSemantics} instance that is equivalent to this one + * for the purpose of caching the methods available on any given Starlark class. + */ + public StarlarkSemantics getStarlarkClassDescriptorCacheKey() { + return this; + } + + @Override + public final int hashCode() { + return hashCode; + } + + @Override + public final boolean equals(Object that) { + return this == that + || (that instanceof StarlarkSemantics && this.map.equals(((StarlarkSemantics) that).map)); + } + + /** + * Returns a representation of this StarlarkSemantics' non-default key/value pairs in key order. + */ + @Override + public final String toString() { + // Print "StarlarkSemantics{k=v, ...}", without +/- prefixes. + StringBuilder buf = new StringBuilder(); + buf.append("StarlarkSemantics{"); + String sep = ""; + for (Map.Entry e : map.entrySet()) { + String key = e.getKey(); + buf.append(sep); + sep = ", "; + if (key.charAt(0) == '+' || key.charAt(0) == '-') { + buf.append(key, 1, key.length()); + } else { + buf.append(key); + } + buf.append('=').append(e.getValue()); + } + return buf.append('}').toString(); + } + + // -- semantics options affecting the Starlark interpreter itself -- + + /** Change the behavior of 'print' statements. Used in tests to verify flag propagation. */ + public static final String PRINT_TEST_MARKER = "-print_test_marker"; + + /** + * Whether recursive function calls are allowed. This option is not exposed to Bazel, which + * unconditionally prohibits recursion. + */ + public static final String ALLOW_RECURSION = "-allow_recursion"; + + /** Whether StarlarkSet objects may be constructed by the interpreter. */ + public static final String EXPERIMENTAL_ENABLE_STARLARK_SET = "-experimental_enable_starlark_set"; +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkSet.java b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkSet.java new file mode 100644 index 000000000..144fefd78 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkSet.java @@ -0,0 +1,777 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import java.util.AbstractSet; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nullable; +import net.starlark.java.annot.Param; +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; + +/** A finite, mutable set of Starlark values. */ +@StarlarkBuiltin( + name = "set", + category = "core", + doc = + """ +Experimental. This API is experimental and may change at any time. Please do not depend on +it. It may be enabled on an experimental basis by setting +--experimental_enable_starlark_set. + +

The built-in mutable set type. Example set expressions: + +

+x = set()           # x is an empty set
+y = set([1, 2, 3])  # y is a set with 3 elements
+3 in y              # True
+0 in y              # False
+len(x)              # 0
+len(y)              # 3
+
+ +

A set used in Boolean context is true if and only if it is non-empty. + +

+s = set()
+"non-empty" if s else "empty"  # "empty"
+t = set(["x", "y"])
+"non-empty" if t else "empty"  # "non-empty"
+
+ +

The elements of a set must be hashable; x may be an element of a set if and only if +x may be used as a key of a dict. + +

A set itself is not hashable; therefore, you cannot have a set with another set as an +element. + +

You cannot access the elements of a set by index, but you can iterate over them, and you can +obtain the list of a set's elements in iteration order using the list() built-in +function. Just like for lists, it is an error to mutate a set while it is being iterated over. The +order of iteration matches insertion order: + +

+s = set([3, 1, 3])
+s.add(2)
+# prints 3, 1, 2
+for item in s:
+    print(item)
+list(s)  # [3, 1, 2]
+
+ +

A set s is equal to t if and only if t is a set containing +the same elements, possibly with a different iteration order. In particular, a set is +not equal to its list of elements. + +

Sets are not ordered; the <, <=, >, and +>= operations are not defined for sets, and a list of sets cannot be sorted - unlike +in Python. + +

The | operation on two sets returns the union of the two sets: a set containing the +elements found in either one or both of the original sets. The | operation has an +augmented assignment version; s |= t adds to s all the elements of +t. + +

+set([1, 2]) | set([3, 2])  # set([1, 2, 3])
+s = set([1, 2])
+s |= set([2, 3, 4])        # s now equals set([1, 2, 3, 4])
+
+ +

The & operation on two sets returns the intersection of the two sets: a set +containing only the elements found in both of the original sets. The & operation +has an augmented assignment version; s &= t removes from s all the +elements not found in t. + +

+set([1, 2]) & set([2, 3])  # set([2])
+set([1, 2]) & set([3, 4])  # set()
+s = set([1, 2])
+s &= set([0, 1])           # s now equals set([1])
+
+ +

The - operation on two sets returns the difference of the two sets: a set containing +the elements found in the left-hand side set but not the right-hand site set. The - +operation has an augmented assignment version; s -= t removes from s all +the elements found in t. + +

+set([1, 2]) - set([2, 3])  # set([1])
+set([1, 2]) - set([3, 4])  # set([1, 2])
+s = set([1, 2])
+s -= set([0, 1])           # s now equals set([2])
+
+ +

The ^ operation on two sets returns the symmetric difference of the two sets: a set +containing the elements found in exactly one of the two original sets, but not in both. The +^ operation has an augmented assignment version; s ^= t removes from +s any element of t found in s and adds to s any +element of t not found in s. + +

+set([1, 2]) ^ set([2, 3])  # set([1, 3])
+set([1, 2]) ^ set([3, 4])  # set([1, 2, 3, 4])
+s = set([1, 2])
+s ^= set([0, 1])           # s now equals set([2, 0])
+
+""") +public final class StarlarkSet extends AbstractSet + implements Mutability.Freezable, StarlarkMembershipTestable, StarlarkIterable { + + private static final StarlarkSet EMPTY = new StarlarkSet<>(ImmutableSet.of()); + + // Either LinkedHashSet or ImmutableSet. + private final Set contents; + // Number of active iterators (unused once frozen). + private transient int iteratorCount; // transient for serialization by Bazel + + /** Final except for {@link #unsafeShallowFreeze}; must not be modified any other way. */ + private Mutability mutability; + + @SuppressWarnings("NonApiType") + private StarlarkSet(Mutability mutability, LinkedHashSet contents) { + checkNotNull(mutability); + checkArgument(mutability != Mutability.IMMUTABLE); + this.mutability = mutability; + this.contents = contents; + } + + private StarlarkSet(ImmutableSet contents) { + // An immutable set might as well store its contents as an ImmutableSet, since ImmutableSet + // both is more memory-efficient than LinkedHashSet and also it has the requisite deterministic + // iteration order. + this.mutability = Mutability.IMMUTABLE; + this.contents = contents; + } + + @Override + public boolean truth() { + return !isEmpty(); + } + + @Override + public boolean isImmutable() { + return mutability().isFrozen(); + } + + @Override + public boolean updateIteratorCount(int delta) { + if (mutability().isFrozen()) { + return false; + } + if (delta > 0) { + iteratorCount++; + } else if (delta < 0) { + iteratorCount--; + } + return iteratorCount > 0; + } + + @Override + public void checkHashable() throws EvalException { + // Even a frozen set is unhashable. + throw Starlark.errorf("unhashable type: 'set'"); + } + + @Override + public int hashCode() { + return contents.hashCode(); + } + + @Override + public void repr(Printer printer) { + if (isEmpty()) { + printer.append("set()"); + } else { + printer.printList(this, "set([", ", ", "])"); + } + } + + @Override + public String toString() { + return Starlark.repr(this); + } + + @Override + public boolean equals(Object o) { + return contents.equals(o); + } + + @Override + public Iterator iterator() { + if (contents instanceof ImmutableSet) { + return contents.iterator(); + } else { + // Prohibit mutation through Iterator.remove(). + return Collections.unmodifiableSet(contents).iterator(); + } + } + + @Override + public int size() { + return contents.size(); + } + + @Override + public boolean isEmpty() { + return contents.isEmpty(); + } + + @Override + public Object[] toArray() { + return contents.toArray(); + } + + @Override + public T[] toArray(T[] a) { + return contents.toArray(a); + } + + @Override + public boolean contains(Object o) { + return contents.contains(o); + } + + @Override + public boolean containsAll(Collection c) { + return contents.containsAll(c); + } + + @Override + public boolean containsKey(StarlarkSemantics semantics, Object element) { + return contents.contains(element); + } + + /** Returns an immutable empty set. */ + // Safe because the empty singleton is immutable. + @SuppressWarnings("unchecked") + public static StarlarkSet empty() { + return (StarlarkSet) EMPTY; + } + + /** Returns a new empty set with the specified mutability. */ + public static StarlarkSet of(@Nullable Mutability mu) { + if (mu == null) { + mu = Mutability.IMMUTABLE; + } + if (mu == Mutability.IMMUTABLE) { + return empty(); + } else { + return new StarlarkSet<>(mu, Sets.newLinkedHashSetWithExpectedSize(1)); + } + } + + /** + * Returns a set with the specified mutability containing the entries of {@code elements}. Tries + * to elide copying if {@code elements} is immutable. + * + * @param elements a collection of elements, which must be Starlark-hashable (note that this + * method assumes but does not verify their hashability), to add to the new set. + */ + public static StarlarkSet copyOf( + @Nullable Mutability mu, Collection elements) { + if (elements.isEmpty()) { + return of(mu); + } + + if (mu == null) { + mu = Mutability.IMMUTABLE; + } + + if (mu == Mutability.IMMUTABLE) { + if (elements instanceof ImmutableSet) { + elements.forEach(Starlark::checkValid); + @SuppressWarnings("unchecked") + ImmutableSet immutableSet = (ImmutableSet) elements; + return new StarlarkSet<>(immutableSet); + } + + if (elements instanceof StarlarkSet && ((StarlarkSet) elements).isImmutable()) { + @SuppressWarnings("unchecked") + StarlarkSet starlarkSet = (StarlarkSet) elements; + return starlarkSet; + } + + ImmutableSet.Builder immutableSetBuilder = + ImmutableSet.builderWithExpectedSize(elements.size()); + elements.forEach(e -> immutableSetBuilder.add(Starlark.checkValid(e))); + return new StarlarkSet<>(immutableSetBuilder.build()); + } else { + LinkedHashSet linkedHashSet = Sets.newLinkedHashSetWithExpectedSize(elements.size()); + elements.forEach(e -> linkedHashSet.add(Starlark.checkValid(e))); + return new StarlarkSet<>(mu, linkedHashSet); + } + } + + private static StarlarkSet wrapOrImmutableCopy(Mutability mu, LinkedHashSet elements) { + checkNotNull(mu); + if (mu == Mutability.IMMUTABLE) { + return elements.isEmpty() ? empty() : new StarlarkSet<>(ImmutableSet.copyOf(elements)); + } else { + return new StarlarkSet<>(mu, elements); + } + } + + /** + * A variant of {@link #copyOf} intended to be used from Starlark. Unlike {@link #copyOf}, this + * method does verify that the elements being added to the set are Starlark-hashable. + * + * @param elements a collection of elements to add to the new set, or a map whose keys will be + * added to the new set. + */ + public static StarlarkSet checkedCopyOf(@Nullable Mutability mu, Object elements) + throws EvalException { + @SuppressWarnings("unchecked") + Collection collection = + (Collection) toHashableCollection(elements, "set constructor argument"); + return copyOf(mu, collection); + } + + /** + * Returns an immutable set containing the entries of {@code elements}. Tries to elide copying if + * {@code elements} is already immutable. + * + * @param elements a collection of elements, which must be Starlark-hashable (note that this + * method assumes but does not verify their hashability), to add to the new set. + */ + public static StarlarkSet immutableCopyOf(Collection elements) { + return copyOf(null, elements); + } + + @Override + public Mutability mutability() { + return mutability; + } + + @Override + public void unsafeShallowFreeze() { + Mutability.Freezable.checkUnsafeShallowFreezePrecondition(this); + this.mutability = Mutability.IMMUTABLE; + } + + @StarlarkMethod( + name = "issubset", + doc = + """ +Returns true of this set is a subset of another. + +

For example, +

+set([1, 2]).issubset([1, 2, 3]) == True
+set([1, 2]).issubset([1, 2]) == True
+set([1, 2]).issubset([2, 3]) == False
+
+""", + parameters = {@Param(name = "other", doc = "A set, sequence, or dict.")}) + public boolean isSubset(Object other) throws EvalException { + return toCollection(other, "issubset argument").containsAll(this.contents); + } + + @StarlarkMethod( + name = "issuperset", + doc = + """ +Returns true of this set is a superset of another. + +

For example, +

+set([1, 2, 3]).issuperset([1, 2]) == True
+set([1, 2, 3]).issuperset([1, 2, 3]) == True
+set([1, 2, 3]).issuperset([2, 3, 4]) == False
+
+""", + parameters = {@Param(name = "other", doc = "A set, sequence, or dict.")}) + public boolean isSuperset(Object other) throws EvalException { + return contents.containsAll(toCollection(other, "issuperset argument")); + } + + @StarlarkMethod( + name = "isdisjoint", + doc = + """ +Returns true if this set has no elements in common with another. + +

For example, +

+set([1, 2]).isdisjoint([3, 4]) == True
+set().isdisjoint(set()) == True
+set([1, 2]).isdisjoint([2, 3]) == False
+
+""", + parameters = {@Param(name = "other", doc = "A set, sequence, or dict.")}) + public boolean isDisjoint(Object other) throws EvalException { + return Collections.disjoint(this.contents, toCollection(other, "isdisjoint argument")); + } + + /** + * Intended for use from Starlark; if used from Java, the caller should ensure that the elements + * to be added are instances of {@code E}. + */ + @StarlarkMethod( + name = "update", + doc = + """ +Adds the elements found in others to this set. + +

For example, +

+x = set([1, 2])
+x.update([2, 3], [3, 4])
+# x is now set([1, 2, 3, 4])
+
+""", + extraPositionals = @Param(name = "others", doc = "Sets, sequences, or dicts.")) + public void update(Tuple others) throws EvalException { + Starlark.checkMutable(this); + for (Object other : others) { + @SuppressWarnings("unchecked") + Collection otherCollection = + (Collection) toHashableCollection(other, "update argument"); + contents.addAll(otherCollection); + } + } + + @StarlarkMethod( + name = "add", + doc = "Adds an element to the set.", + parameters = {@Param(name = "element", doc = "Element to add.")}) + public void addElement(E element) throws EvalException { + Starlark.checkMutable(this); + Starlark.checkHashable(element); + contents.add(element); + } + + @StarlarkMethod( + name = "remove", + doc = + """ +Removes an element, which must be present in the set, from the set. Fails if the element was not +present in the set. +""", + parameters = {@Param(name = "element", doc = "Element to remove.")}) + public void removeElement(E element) throws EvalException { + Starlark.checkMutable(this); + if (!contents.remove(element)) { + throw Starlark.errorf("element %s not found in set", Starlark.repr(element)); + } + } + + @StarlarkMethod( + name = "discard", + doc = "Removes an element from the set if it is present.", + parameters = {@Param(name = "element", doc = "Element to discard.")}) + public void discard(E element) throws EvalException { + Starlark.checkMutable(this); + contents.remove(element); + } + + @StarlarkMethod( + name = "pop", + doc = "Removes and returns the first element of the set. Fails if the set is empty.") + public E pop() throws EvalException { + Starlark.checkMutable(this); + if (isEmpty()) { + throw Starlark.errorf("set is empty"); + } + E element = contents.iterator().next(); + contents.remove(element); + return element; + } + + @StarlarkMethod(name = "clear", doc = "Removes all the elements of the set.") + public void clearElements() throws EvalException { + Starlark.checkMutable(this); + contents.clear(); + } + + @StarlarkMethod( + name = "union", + doc = + """ +Returns a new mutable set containing the union of this set with others. + +

For example, +

+set([1, 2]).union([2, 3, 4], [4, 5]) == set([1, 2, 3, 4, 5])
+
+""", + extraPositionals = @Param(name = "others", doc = "Sets, sequences, or dicts."), + useStarlarkThread = true) + public StarlarkSet union(Tuple others, StarlarkThread thread) throws EvalException { + LinkedHashSet newContents = new LinkedHashSet<>(contents); + for (Object other : others) { + newContents.addAll(toHashableCollection(other, "union argument")); + } + return wrapOrImmutableCopy(thread.mutability(), newContents); + } + + @StarlarkMethod( + name = "intersection", + doc = + """ +Returns a new mutable set containing the intersection of this set with others. + +

For example, +

+set([1, 2, 3]).intersection([1, 2], [2, 3]) == set([2])
+
+""", + extraPositionals = @Param(name = "others", doc = "Sets, sequences, or dicts."), + useStarlarkThread = true) + public StarlarkSet intersection(Tuple others, StarlarkThread thread) throws EvalException { + LinkedHashSet newContents = new LinkedHashSet<>(contents); + for (Object other : others) { + newContents.retainAll(toCollection(other, "intersection argument")); + } + return wrapOrImmutableCopy(thread.mutability(), newContents); + } + + @StarlarkMethod( + name = "intersection_update", + doc = + """ +Removes any elements not found in all others from this set. + +

For example, +

+x = set([1, 2, 3, 4])
+x.intersection_update([2, 3], [3, 4])
+# x is now set([3])
+
+""", + extraPositionals = @Param(name = "others", doc = "Sets, sequences, or dicts.")) + public void intersectionUpdate(Tuple others) throws EvalException { + Starlark.checkMutable(this); + for (Object other : others) { + contents.retainAll(toCollection(other, "intersection_update argument")); + } + } + + @StarlarkMethod( + name = "difference", + doc = + """ +Returns a new mutable set containing the difference of this set with others. + +

For example, +

+set([1, 2, 3]).intersection([1, 2], [2, 3]) == set([2])
+
+""", + extraPositionals = @Param(name = "others", doc = "Sets, sequences, or dicts."), + useStarlarkThread = true) + public StarlarkSet difference(Tuple others, StarlarkThread thread) throws EvalException { + LinkedHashSet newContents = new LinkedHashSet<>(contents); + for (Object other : others) { + newContents.removeAll(toCollection(other, "difference argument")); + } + return wrapOrImmutableCopy(thread.mutability(), newContents); + } + + @StarlarkMethod( + name = "difference_update", + doc = + """ +Removes any elements found in any others from this set. + +

For example, +

+x = set([1, 2, 3, 4])
+x.difference_update([2, 3], [3, 4])
+# x is now set([1])
+
+""", + extraPositionals = @Param(name = "others", doc = "Sets, sequences, or dicts.")) + public void differenceUpdate(Tuple others) throws EvalException { + Starlark.checkMutable(this); + for (Object other : others) { + contents.removeAll(toCollection(other, "intersection_update argument")); + } + } + + @StarlarkMethod( + name = "symmetric_difference", + doc = + """ +Returns a new mutable set containing the symmetric difference of this set with another set, +sequence, or dict. + +

For example, +

+set([1, 2, 3]).symmetric_difference([2, 3, 4]) == set([1, 4])
+
+""", + parameters = {@Param(name = "other", doc = "A set, sequence, or dict.")}, + useStarlarkThread = true) + public StarlarkSet symmetricDifference(Object other, StarlarkThread thread) + throws EvalException { + LinkedHashSet newContents = new LinkedHashSet<>(contents); + for (Object element : toHashableCollection(other, "symmetric_difference argument")) { + if (contents.contains(element)) { + newContents.remove(element); + } else { + newContents.add(element); + } + } + return wrapOrImmutableCopy(thread.mutability(), newContents); + } + + /** + * Intended for use from Starlark; if used from Java, the caller should ensure that the elements + * to be added are instances of {@code E}. + */ + @StarlarkMethod( + name = "symmetric_difference_update", + doc = + """ +Returns a new mutable set containing the symmetric difference of this set with another set, +sequence, or dict. + +

For example, +

+set([1, 2, 3]).symmetric_difference([2, 3, 4]) == set([1, 4])
+
+""", + parameters = {@Param(name = "other", doc = "A set, sequence, or dict.")}) + public void symmetricDifferenceUpdate(Object other) throws EvalException { + Starlark.checkMutable(this); + ImmutableSet originalContents = ImmutableSet.copyOf(contents); + for (Object element : toHashableCollection(other, "symmetric_difference_update argument")) { + if (originalContents.contains(element)) { + contents.remove(element); + } else { + @SuppressWarnings("unchecked") + E castElement = (E) element; + contents.add(castElement); + } + } + } + + /** + * Verifies that {@code other} is either a collection or a map. + * + * @return {@code other} if it is a collection, or the key set of {@code other} if it is a map. + */ + private static Collection toCollection(Object other, String what) throws EvalException { + if (other instanceof Collection) { + return (Collection) other; + } else if (other instanceof Map) { + return ((Map) other).keySet(); + } + throw notSizedIterableError(other, what); + } + + /** + * A variant of {@link #toCollection} which additionally checks whether the returned collection's + * elements are Starlark-hashable. + * + * @return {@code other} if it is a collection, or the key set of {@code other} if it is a map. + */ + private static Collection toHashableCollection(Object other, String what) + throws EvalException { + if (other instanceof Collection) { + Collection collection = (Collection) other; + // Assume that elements of a StarlarkSet have already been checked to be hashable. + if (!(collection instanceof StarlarkSet)) { + for (Object element : collection) { + Starlark.checkHashable(element); + } + } + return collection; + } else if (other instanceof Map) { + Set keySet = ((Map) other).keySet(); + // Assume that keys of a Dict have already been checked to be hashable. + if (!(other instanceof Dict)) { + for (Object element : keySet) { + Starlark.checkHashable(element); + } + } + return keySet; + } + throw notSizedIterableError(other, what); + } + + // Starlark doesn't have a "sized iterable" interface - so we enumerate the types we expect. + private static EvalException notSizedIterableError(Object other, String what) { + return Starlark.errorf( + "for %s got value of type '%s', want a set, sequence, or dict", what, Starlark.type(other)); + } + + // Prohibit Java Set mutators. + + /** + * @deprecated use {@link #addElement} instead. + */ + @Deprecated + @Override + public boolean add(E e) { + throw new UnsupportedOperationException(); + } + + /** + * @deprecated use {@link #update} instead. + */ + @Deprecated + @Override + public boolean addAll(Collection c) { + throw new UnsupportedOperationException(); + } + + /** + * @deprecated use {@link #clearElements} instead. + */ + @Deprecated + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + /** + * @deprecated use {@link #removeElement} instead. + */ + @Deprecated + @Override + public boolean remove(Object o) { + throw new UnsupportedOperationException(); + } + + /** + * @deprecated use {@link #differenceUpdate} instead. + */ + @Deprecated + @Override + public boolean removeAll(Collection c) { + throw new UnsupportedOperationException(); + } + + /** + * @deprecated use {@link #intersectionUpdate} instead. + */ + @Deprecated + @Override + public boolean retainAll(Collection c) { + throw new UnsupportedOperationException(); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkThread.java b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkThread.java new file mode 100644 index 000000000..b073a8bd6 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkThread.java @@ -0,0 +1,615 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; +import net.starlark.java.syntax.Location; + +/** + * An StarlarkThread represents a Starlark thread. + * + *

It holds the stack of active Starlark and built-in function calls. In addition, it may hold + * per-thread application state (see {@link #setThreadLocal}) that passes through Starlark functions + * but does not directly affect them, such as information about the BUILD file being loaded. + * + *

StarlarkThreads are not thread-safe: they should be confined to a single Java thread. + * + *

Every StarlarkThread has an associated {@link Mutability}, which should be created for that + * thread, and closed once the thread's work is done. (A try-with-resources statement is handy for + * this purpose.) Starlark values created by the thread are associated with the thread's Mutability, + * so that when the Mutability is closed at the end of the computation, all the values created by + * the thread become frozen. This pattern ensures that all Starlark values are frozen before they + * are published to another thread, and thus that concurrently executing Starlark threads are free + * from data races. Once a thread's mutability is frozen, the thread is unlikely to be useful for + * further computation because it can no longer create mutable values. (This is occasionally + * valuable in tests.) + */ +public final class StarlarkThread { + + /** The mutability of values created by this thread. */ + private final Mutability mutability; + + // profiler state + // + // The profiler field (and savedThread) are set when we first observe during a + // push (function call entry) that the profiler is active. They are unset + // not in the corresponding pop, but when the last frame is popped, because + // the profiler session might start in the middle of a call and/or run beyond + // the lifetime of this thread. + final AtomicInteger cpuTicks = new AtomicInteger(); + @Nullable private CpuProfiler profiler; + private StarlarkThread savedThread; // saved StarlarkThread, when profiling reentrant evaluation + + private final Map, Object> threadLocals = new HashMap<>(); + + private final SymbolGenerator symbolGenerator; + + private boolean interruptible = true; + + long steps; // count of logical computation steps executed so far + long stepLimit = Long.MAX_VALUE; // limit on logical computation steps + + /** + * Returns the number of Starlark computation steps executed by this thread according to a + * small-step semantics. (Today, that means exec, eval, and assign operations executed by the + * tree-walking evaluator, but in future will mean byte code instructions; the two are not + * commensurable.) + */ + public long getExecutedSteps() { + return steps; + } + + /** + * Sets the maximum number of Starlark computation steps that may be executed by this thread (see + * {@link #getExecutedSteps}). When the step counter reaches or exceeds this value, execution + * fails with an EvalException. + */ + public void setMaxExecutionSteps(long steps) { + this.stepLimit = steps; + } + + /** + * Disables polling of the {@link java.lang.Thread#interrupted} flag during Starlark evaluation. + */ + // TODO(adonovan): expose a public API for this if we can establish a stronger semantics. (There + // are other ways besides polling for evaluation to be interrupted, such as calling certain + // built-in functions.) + void ignoreThreadInterrupts() { + interruptible = false; + } + + void checkInterrupt() throws InterruptedException { + if (interruptible && Thread.interrupted()) { + throw new InterruptedException(); + } + } + + /** + * setThreadLocal saves {@code value} as a thread-local variable of this Starlark thread, keyed by + * {@code key}, so that it can later be retrieved by {@code getThreadLocal(key)}. + */ + public void setThreadLocal(Class key, T value) { + threadLocals.put(key, value); + } + + /** + * getThreadLocal returns the value {@code v} supplied to the most recent {@code + * setThreadLocal(key, v)} call, or null if there was no prior call. + */ + public T getThreadLocal(Class key) { + Object v = threadLocals.get(key); + return v == null ? null : key.cast(v); + } + + /** A Frame records information about an active function call. */ + static final class Frame implements Debug.Frame { + final StarlarkThread thread; + final StarlarkCallable fn; // the called function + + @Nullable + final Debug.Debugger dbg = Debug.debugger.get(); // the debugger, if active for this frame + + Object result = Starlark.NONE; // the operand of a Starlark return statement + + // Current PC location. Initially fn.getLocation(); for Starlark functions, + // it is updated at key points when it may be observed: calls, breakpoints, errors. + private Location loc; + + // Indicates that setErrorLocation has been called already and the error + // location (loc) should not be overrwritten. + private boolean errorLocationSet; + + // The locals of this frame, if fn is a StarlarkFunction, otherwise null. + // Set by StarlarkFunction.fastcall. Elements may be regular Starlark + // values, or wrapped in StarlarkFunction.Cells if shared with a nested function. + @Nullable Object[] locals; + + private long profileStartTimeNanos; // start time nanos of walltime call profiler + + private Frame(StarlarkThread thread, StarlarkCallable fn) { + this.thread = thread; + this.fn = fn; + } + + // Updates the PC location in this frame. + void setLocation(Location loc) { + this.loc = loc; + } + + // Sets location only the first time it is called, + // to ensure that the location of the innermost expression + // is used for errors. + // (Once we switch to a bytecode interpreter, we can afford + // to update fr.pc before each fallible operation, but until then + // we must materialize Locations only after the fact of failure.) + // Sets errorLocationSet. + void setErrorLocation(Location loc) { + if (!errorLocationSet) { + errorLocationSet = true; + this.loc = loc; + } + } + + @Override + public StarlarkCallable getFunction() { + return fn; + } + + @Override + public Location getLocation() { + return loc; + } + + @Override + public ImmutableMap getLocals() { + // TODO(adonovan): provide a more efficient API. + ImmutableMap.Builder env = ImmutableMap.builder(); + if (fn instanceof StarlarkFunction) { + for (int i = 0; i < locals.length; i++) { + Object local = locals[i]; + if (local instanceof StarlarkFunction.Cell) { + local = ((StarlarkFunction.Cell) local).x; + } + if (local != null) { + env.put(((StarlarkFunction) fn).rfn.getLocals().get(i).getName(), local); + } + } + } + return env.buildOrThrow(); + } + + @Override + public String toString() { + return fn.getName() + "@" + loc; + } + } + + /** The semantics options that affect how Starlark code is evaluated. */ + private final StarlarkSemantics semantics; + + /** Whether recursive calls are allowed (cached from semantics). */ + private final boolean allowRecursion; + + /** PrintHandler for Starlark print statements. */ + private PrintHandler printHandler = StarlarkThread::defaultPrintHandler; + + /** Loader for Starlark load statements. Null if loading is disallowed. */ + @Nullable private Loader loader = null; + + private UncheckedExceptionContext uncheckedExceptionContext = () -> ""; + + /** Stack of active function calls. */ + private final ArrayList callstack = new ArrayList<>(); + + /** A hook for notifications of assignments at top level. */ + PostAssignHook postAssignHook; + + /** Pushes a function onto the call stack. */ + void push(StarlarkCallable fn) { + Frame fr = new Frame(this, fn); + callstack.add(fr); + + // Notify debug tools of the thread's first push. + if (callstack.size() == 1 && Debug.threadHook != null) { + Debug.threadHook.onPushFirst(this); + } + + fr.loc = fn.getLocation(); + + // Start wall-time call profile span. + CallProfiler callProfiler = StarlarkThread.callProfiler; + if (callProfiler != null) { + fr.profileStartTimeNanos = callProfiler.start(); + } + + // Poll for newly installed CPU profiler. + if (profiler == null) { + this.profiler = CpuProfiler.get(); + if (profiler != null) { + cpuTicks.set(0); + // Associated current Java thread with this StarlarkThread. + // (Save the previous association so we can restore it later.) + this.savedThread = CpuProfiler.setStarlarkThread(this); + } + } + } + + /** Pops a function off the call stack. */ + void pop() { + int last = callstack.size() - 1; + Frame fr = callstack.get(last); + + if (profiler != null) { + int ticks = cpuTicks.getAndSet(0); + if (ticks > 0) { + profiler.addEvent(ticks, getDebugCallStack()); + } + + // If this is the final pop in this thread, + // unregister it from the profiler. + if (last == 0) { + // Restore the previous association (in case of reentrant evaluation). + CpuProfiler.setStarlarkThread(this.savedThread); + this.savedThread = null; + this.profiler = null; + } + } + + callstack.remove(last); // pop + + // End wall-time profile span. + CallProfiler callProfiler = StarlarkThread.callProfiler; + if (callProfiler != null && fr.profileStartTimeNanos >= 0) { + callProfiler.end(fr.profileStartTimeNanos, fr.fn); + } + + // Notify debug tools of the thread's last pop. + if (last == 0 && Debug.threadHook != null) { + Debug.threadHook.onPopLast(this); + } + } + + /** Returns the mutability for values created by this thread. */ + public Mutability mutability() { + return mutability; + } + + /** + * A PrintHandler determines how a Starlark thread deals with print statements. It is invoked by + * the built-in {@code print} function. Its default behavior is to write the message to standard + * error, preceded by the location of the print statement, {@code thread.getCallerLocation()}. + */ + @FunctionalInterface + public interface PrintHandler { + void print(StarlarkThread thread, String msg); + } + + /** Returns the PrintHandler for Starlark print statements. */ + PrintHandler getPrintHandler() { + return printHandler; + } + + /** Sets the behavior of Starlark print statements executed by this thread. */ + public void setPrintHandler(PrintHandler h) { + this.printHandler = Preconditions.checkNotNull(h); + } + + private static void defaultPrintHandler(StarlarkThread thread, String msg) { + System.err.println(thread.getCallerLocation() + ": " + msg); + } + + /** + * A Loader determines the behavior of load statements executed by this thread. It returns the + * named module, or null if not found. + */ + @FunctionalInterface + public interface Loader { + @Nullable + Module load(String module); + } + + /** Returns the loader for Starlark load statements. */ + Loader getLoader() { + return loader; + } + + /** Sets the behavior of Starlark load statements executed by this thread. */ + public void setLoader(Loader loader) { + this.loader = Preconditions.checkNotNull(loader); + } + + /** + * Supplies additional context to append to the message of {@link Starlark.UncheckedEvalException} + * or {@link Starlark.UncheckedEvalError}. + */ + // TODO(brandjon): This seems unnecessary. Instead of implementing a hook that is mutated after + // thread is constructed, we should be able to just attach this information at construction time. + public interface UncheckedExceptionContext { + String getContextForUncheckedException(); + } + + public void setUncheckedExceptionContext(UncheckedExceptionContext uncheckedExceptionContext) { + this.uncheckedExceptionContext = Preconditions.checkNotNull(uncheckedExceptionContext); + } + + public String getContextDescription() { + return uncheckedExceptionContext.getContextForUncheckedException(); + } + + /** Reports whether {@code fn} has been recursively reentered within this thread. */ + boolean isRecursiveCall(StarlarkFunction fn) { + // Find fn buried within stack. (The top of the stack is assumed to be fn.) + for (int i = callstack.size() - 2; i >= 0; --i) { + Frame fr = callstack.get(i); + // We compare code, not closure values, otherwise one can defeat the + // check by writing the Y combinator. + if (fr.fn instanceof StarlarkFunction && ((StarlarkFunction) fr.fn).rfn.equals(fn.rfn)) { + return true; + } + } + return false; + } + + /** + * Returns the location of the program counter in the enclosing call frame. If called from within + * a built-in function, this is the location of the call expression that called the built-in. It + * returns BUILTIN if called with fewer than two frames (such as within a test). + */ + public Location getCallerLocation() { + return toplevel() ? Location.BUILTIN : frame(1).loc; + } + + /** + * Reports whether the call stack has less than two frames. Zero frames means an idle thread. One + * frame means the function for the top-level statements of a file is active. More than that means + * a function call is in progress. + * + *

Every use of this function is a hack to work around the lack of proper local vs global + * identifier resolution at top level. + */ + private boolean toplevel() { + return callstack.size() < 2; + } + + // Returns the stack frame at the specified depth. 0 means top of stack, 1 is its caller, etc. + Frame frame(int depth) { + return callstack.get(callstack.size() - 1 - depth); + } + + /** + * Creates a StarlarkThread. + * + * @param mu the (non-frozen) mutability of values created by this thread. + * @param semantics the StarlarkSemantics for this thread. Note that it is generally a code smell + * to use {@link StarlarkSemantics#DEFAULT} if the application permits customizing the + * semantics (e.g. via command line flags). Usually, all Starlark evaluation contexts within + * the same application would use the same {@code StarlarkSemantics} instance. + * @param contextDescription a short description of this evaluation, added as context when an + * exception is thrown. The empty String can be used as a default value. + * @param symbolGenerator a supplier of deterministic, stable IDs for objects created by this + * thread + */ + // TODO(bazel-team): Consider merging contextDescription into the symbolGenerator. + public static StarlarkThread create( + Mutability mu, + StarlarkSemantics semantics, + String contextDescription, + SymbolGenerator symbolGenerator) { + return new StarlarkThread(mu, semantics, contextDescription, symbolGenerator); + } + + /** + * Creates a StarlarkThread with an empty {@code contextDescription} and transient {@code + * symbolGenerator}. + * + *

See comments at {@link SymbolGenerator#createTransient} for when this is applicable. + */ + public static StarlarkThread createTransient(Mutability mu, StarlarkSemantics semantics) { + return new StarlarkThread( + mu, semantics, /* contextDescription= */ "", SymbolGenerator.createTransient()); + } + + private StarlarkThread( + Mutability mu, + StarlarkSemantics semantics, + String contextDescription, + SymbolGenerator symbolGenerator) { + Preconditions.checkArgument(!mu.isFrozen()); + this.mutability = mu; + this.semantics = semantics; + this.allowRecursion = semantics.getBool(StarlarkSemantics.ALLOW_RECURSION); + if (!contextDescription.isEmpty()) { + setUncheckedExceptionContext(() -> contextDescription); + } + this.symbolGenerator = symbolGenerator; + } + + /** + * Specifies a hook function to be run after each assignment at top level. + * + *

This is a short-term hack to allow us to consolidate all StarlarkFile execution in one place + * even while BzlLoadFunction implements the old "export" behavior, in which rules, aspects and + * providers are "exported" as soon as they are assigned, not at the end of file execution. + */ + public void setPostAssignHook(PostAssignHook postAssignHook) { + this.postAssignHook = postAssignHook; + } + + /** A hook for notifications of assignments at top level. */ + @FunctionalInterface + public interface PostAssignHook { + void assign(String name, Object value); + } + + public StarlarkSemantics getSemantics() { + return semantics; + } + + /** Reports whether this thread is allowed to make recursive calls. */ + boolean isRecursionAllowed() { + return allowRecursion; + } + + // Implementation of Debug.getCallStack. + // Intentionally obscured to steer most users to the simpler getCallStack. + ImmutableList getDebugCallStack() { + return ImmutableList.copyOf(callstack); + } + + @Nullable + StarlarkFunction getInnermostEnclosingStarlarkFunction(int depth) { + Preconditions.checkArgument(depth >= 0); + for (int i = callstack.size() - 1; i >= 0; i--) { + Debug.Frame fr = callstack.get(i); + if (fr.getFunction() instanceof StarlarkFunction) { + if (depth == 0) { + return (StarlarkFunction) fr.getFunction(); + } + depth--; + } + } + return null; + } + + /** Returns the size of the callstack. This is needed for the debugger. */ + int getCallStackSize() { + return callstack.size(); + } + + /** + * The value of {@link CallStackEntry#name} for the implicit function that executes the top-level + * statements of a file. + */ + public static final String TOP_LEVEL = ""; + + /** Creates a new {@link CallStackEntry}. */ + public static CallStackEntry callStackEntry(String name, Location location) { + return new CallStackEntry(name, location); + } + + /** + * A CallStackEntry describes the name and PC location of an active function call. See {@link + * #getCallStack}. + */ + @Immutable + public static final class CallStackEntry { + public final String name; + public final Location location; + + private CallStackEntry(String name, Location location) { + this.name = Preconditions.checkNotNull(name); + this.location = Preconditions.checkNotNull(location); + } + + @Override + public String toString() { + return name + "@" + location; + } + + @Override + public int hashCode() { + return 31 * name.hashCode() + location.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CallStackEntry)) { + return false; + } + CallStackEntry that = (CallStackEntry) o; + return name.equals(that.name) && location.equals(that.location); + } + } + + /** + * Returns information about this thread's current stack of active function calls, outermost call + * first. For each function, it reports its name, and the location of its current program counter. + * The result is immutable and does not reference interpreter data structures, so it may retained + * indefinitely and safely shared with other threads. + */ + public ImmutableList getCallStack() { + ImmutableList.Builder stack = + ImmutableList.builderWithExpectedSize(callstack.size()); + for (Frame fr : callstack) { + stack.add(callStackEntry(fr.fn.getName(), fr.loc)); + } + return stack.build(); + } + + /** Sets the given throwable's stack trace to a Java-style version of {@link #getCallStack}. */ + void fillInStackTrace(Throwable throwable) { + StackTraceElement[] trace = new StackTraceElement[callstack.size()]; + for (int i = 0; i < callstack.size(); i++) { + Frame frame = callstack.get(i); + trace[trace.length - i - 1] = + new StackTraceElement( + "", frame.fn.getName(), frame.loc.file(), frame.loc.line()); + } + throwable.setStackTrace(trace); + } + + @Override + public int hashCode() { + throw new UnsupportedOperationException(); // avoid nondeterminism + } + + @Override + public boolean equals(Object that) { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + return String.format("", mutability); + } + + /** CallProfiler records the start and end wall times of function calls. */ + public interface CallProfiler { + long start(); + + @SuppressWarnings("GoodTime") // This code is very performance sensitive. + void end(long startTimeNanos, StarlarkCallable fn); + } + + /** Installs a global hook that will be notified of function calls. */ + public static void setCallProfiler(@Nullable CallProfiler p) { + callProfiler = p; + } + + public SymbolGenerator.Symbol getNextIdentityToken() { + return symbolGenerator.generate(); + } + + public SymbolGenerator getSymbolGenerator() { + return symbolGenerator; + } + + Object getOwner() { + return symbolGenerator.getOwner(); + } + + @Nullable private static CallProfiler callProfiler = null; +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkValue.java b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkValue.java new file mode 100644 index 000000000..c4ae15ef7 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/StarlarkValue.java @@ -0,0 +1,106 @@ +// Copyright 2015 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +/** Base interface for all Starlark values besides boxed Java primitives. */ +public interface StarlarkValue { + + /** + * Prints an official representation of object x. + * + *

Convention is that the string should be parseable back to the value x. If this isn't + * feasible then it should be a short human-readable description enclosed in angled brackets, e.g. + * {@code ""}. + * + * @param printer a printer to be used for formatting nested values. + */ + // TODO(brandjon): We can consider adding a StarlarkSemantics param to repr(), to match str() and + // debugPrint(). Counterargument is that it's nice to have a supported way of stringifying a + // Starlark value independently of a Starlark evaluation environment. If necessary we could use + // toString for that purpose, or even define a separate semantics-independent repr-like method. + default void repr(Printer printer) { + printer.append(""); + } + + /** + * Prints an informal, human-readable representation of the value. + * + *

By default dispatches to the {@code repr} method. + * + * @param printer a printer to be used for formatting nested values. + */ + default void str(Printer printer, StarlarkSemantics semantics) { + repr(printer); + } + + /** + * Prints an informal debug representation of the value. + * + *

This debug representation is only ever printed to the terminal or to another out-of-band + * channel, and is never accessible to Starlark code. Therefore, it is safe for the debug + * representation to reveal properties of the value that are usually hidden for the sake of + * performance, determinism, or forward-compatibility. + * + *

By default dispatches to the {@code str} method. + * + * @param printer a printer to be used for formatting nested values. + */ + default void debugPrint(Printer printer, StarlarkThread thread) { + str(printer, thread.getSemantics()); + } + + /** Returns the truth-value of this Starlark value. */ + default boolean truth() { + return true; + } + + /** Reports whether the value is deeply immutable. */ + // TODO(adonovan): eliminate this concept. All uses really need to know is, is it hashable?, + // because Starlark values must have stable hashes: a hashable value must either be immutable or + // its hash must be part of its identity. + // But this must wait until --incompatible_disallow_hashing_frozen_mutables=true is removed. + // (see github.com/bazelbuild/bazel/issues/7800) + default boolean isImmutable() { + return false; + } + + /** + * Returns normally if the Starlark value is hashable and thus suitable as a dict key. + * + *

(A StarlarkValue implementation may define hashCode and equals and thus be a valid + * java.util.Map key without being hashable by Starlark code.) + * + * @throws EvalException otherwise. + */ + default void checkHashable() throws EvalException { + // Bazel makes widespread assumptions that all Starlark values can be hashed + // by Java code, so we cannot implement checkHashable by having + // StarlarkValue.hashCode throw an unchecked exception, which would be more + // efficient. Instead, before inserting a value in a dict, we must first check + // whether it is hashable by calling this function, and then call its hashCode + // method only if so. + // For structs and tuples, this unfortunately visits the object graph twice. + // + // One subtlety: Bazel's lib.packages.StarlarkInfo.checkHashable, by using this + // default implementation of checkHashable, which is based on isImmutable, + // recursively asks whether its elements are immutable, not hashable. + // Consequently, even though a list may not be used as a dict key (even if frozen), + // a struct containing a list is hashable. + // TODO(adonovan): fix this inconsistency. Requires a Bazel incompatible change. + if (!this.isImmutable()) { + throw Starlark.errorf("unhashable type: '%s'", Starlark.type(this)); + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StringModule.java b/third_party/bazel/main/java/net/starlark/java/eval/StringModule.java new file mode 100644 index 000000000..3d481e501 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/StringModule.java @@ -0,0 +1,1067 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.base.Ascii; +import com.google.common.base.CharMatcher; +import com.google.common.base.Joiner; +import java.util.ArrayList; +import java.util.Collections; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import net.starlark.java.annot.Param; +import net.starlark.java.annot.ParamType; +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; + +/** + * Starlark String module. + * + *

This module has special treatment in Starlark, as its methods represent methods present for + * any 'string' objects in the language. + * + *

Methods of this class annotated with {@link StarlarkMethod} must have a positional-only + * 'String self' parameter as the first parameter of the method. + */ +@StarlarkBuiltin( + name = "string", + category = "core", + doc = + "A language built-in type to support strings. " + + "Examples of string literals:
" + + "

a = 'abc\\ndef'\n"
+            + "b = \"ab'cd\"\n"
+            + "c = \"\"\"multiline string\"\"\"\n"
+            + "\n"
+            + "# Strings support slicing (negative index starts from the end):\n"
+            + "x = \"hello\"[2:4]  # \"ll\"\n"
+            + "y = \"hello\"[1:-1]  # \"ell\"\n"
+            + "z = \"hello\"[:4]  # \"hell\"\n"
+            + "# Slice steps can be used, too:\n"
+            + "s = \"hello\"[::2] # \"hlo\"\n"
+            + "t = \"hello\"[3:0:-1] # \"lle\"\n
" + + "Strings are not directly iterable, use the .elems() " + + "method to iterate over their characters. Examples:
" + + "
\"bc\" in \"abcd\"   # evaluates to True\n"
+            + "x = [c for c in \"abc\".elems()]  # x == [\"a\", \"b\", \"c\"]
\n" + + "Implicit concatenation of strings is not allowed; use the + " + + "operator instead. Comparison operators perform a lexicographical comparison; " + + "use == to test for equality.") +final class StringModule implements StarlarkValue { + + static final StringModule INSTANCE = new StringModule(); + + private StringModule() {} + + // Returns s[start:stop:step], as if by Sequence.getSlice. + static String slice(String s, int start, int stop, int step) throws EvalException { + RangeList indices = new RangeList(start, stop, step); + int n = indices.size(); + if (n == 0) { + return ""; + } else if (n == 1) { + return memoizedCharToString(s.charAt(indices.at(0))); + } else if (step == 1) { // common case + return s.substring(indices.at(0), indices.at(n)); + } else { + char[] res = new char[n]; + for (int i = 0; i < n; ++i) { + res[i] = s.charAt(indices.at(i)); + } + return new String(res); + } + } + + // Nearly all chars in Starlark strings are ASCII. + // This is a cache of single-char strings to avoid allocation in the s[i] operation. + private static final String[] ASCII_CHAR_STRINGS = initCharStrings(); + + private static String[] initCharStrings() { + String[] a = new String[0x80]; + for (int i = 0; i < a.length; ++i) { + a[i] = String.valueOf((char) i); + } + return a; + } + + /** Semantically equivalent to {@link String#valueOf(char)} but faster for ASCII strings. */ + static String memoizedCharToString(char c) { + if (c < ASCII_CHAR_STRINGS.length) { + return ASCII_CHAR_STRINGS[c]; + } else { + return String.valueOf(c); + } + } + + // Returns the substring denoted by str[start:end], which is never out of bounds. + // For speed, we don't return str.substring(start, end), as substring allocates a copy. + // Instead we return the (start, end) indices, packed into the lo/hi arms of a long. + private static long substringIndices(String str, Object start, Object end) throws EvalException { + // This function duplicates the logic of Starlark.slice for strings. + int n = str.length(); + int istart = 0; + if (start != Starlark.NONE) { + istart = EvalUtils.toIndex(Starlark.toInt(start, "start"), n); + } + int iend = n; + if (end != Starlark.NONE) { + iend = EvalUtils.toIndex(Starlark.toInt(end, "end"), n); + } + if (iend < istart) { + iend = istart; // => empty result + } + return pack(istart, iend); // = str.substring(start, end) + } + + private static long pack(int lo, int hi) { + return (((long) hi) << 32) | (lo & 0xffffffffL); + } + + private static int lo(long x) { + return (int) x; + } + + private static int hi(long x) { + return (int) (x >>> 32); + } + + @StarlarkMethod( + name = "join", + doc = + "Returns a string in which the string elements of the argument have been " + + "joined by this string as a separator. Example:
" + + "
\"|\".join([\"a\", \"b\", \"c\"]) == \"a|b|c\""
+              + "
", + parameters = {@Param(name = "self"), @Param(name = "elements", doc = "The objects to join.")}, + useStarlarkThread = true) + public String join(String self, Object elements, StarlarkThread thread) throws EvalException { + Iterable items = Starlark.toIterable(elements); + int i = 0; + for (Object item : items) { + if (!(item instanceof String)) { + throw Starlark.errorf( + "expected string for sequence element %d, got '%s' of type %s", + i, Starlark.str(item, thread.getSemantics()), Starlark.type(item)); + } + i++; + } + return Joiner.on(self).join(items); + } + + @StarlarkMethod( + name = "lower", + doc = "Returns the lower case version of this string.", + parameters = {@Param(name = "self")}) + public String lower(String self) { + return Ascii.toLowerCase(self); + } + + @StarlarkMethod( + name = "upper", + doc = "Returns the upper case version of this string.", + parameters = {@Param(name = "self")}) + public String upper(String self) { + return Ascii.toUpperCase(self); + } + + /** + * For consistency with Python we recognize the same whitespace characters as they do over the + * range 0x00-0xFF. See https://hg.python.org/cpython/file/3.6/Objects/unicodetype_db.h#l5738 This + * list is a consequence of Unicode character information. + * + *

Note that this differs from Python 2.7, which uses ctype.h#isspace(), and from + * java.lang.Character#isWhitespace(), which does not recognize U+00A0. + */ + // TODO(https://github.com/bazelbuild/starlark/issues/112): use the Unicode definition of + // whitespace, matching Python 3. + private static final CharMatcher LATIN1_WHITESPACE = + CharMatcher.anyOf( + "\u0009" + "\n" + "\u000B" + "\u000C" + "\r" + "\u001C" + "\u001D" + "\u001E" + "\u001F " + + "\u0085" + "\u00A0"); + + private static String stringLStrip(String self, CharMatcher matcher) { + for (int i = 0; i < self.length(); i++) { + if (!matcher.matches(self.charAt(i))) { + return self.substring(i); + } + } + return ""; // All characters were stripped. + } + + private static String stringRStrip(String self, CharMatcher matcher) { + for (int i = self.length() - 1; i >= 0; i--) { + if (!matcher.matches(self.charAt(i))) { + return self.substring(0, i + 1); + } + } + return ""; // All characters were stripped. + } + + private static String stringStrip(String self, CharMatcher matcher) { + return stringLStrip(stringRStrip(self, matcher), matcher); + } + + @StarlarkMethod( + name = "lstrip", + doc = + "Returns a copy of the string where leading characters that appear in " + + "chars are removed. Note that chars " + + "is not a prefix: all combinations of its value are removed:" + + "

"
+              + "\"abcba\".lstrip(\"ba\") == \"cba\""
+              + "
", + parameters = { + @Param(name = "self"), + @Param( + name = "chars", + allowedTypes = { + @ParamType(type = String.class), + @ParamType(type = NoneType.class), + }, + doc = "The characters to remove, or all whitespace if None.", + defaultValue = "None") + }) + public String lstrip(String self, Object charsOrNone) { + CharMatcher matcher = + charsOrNone != Starlark.NONE ? CharMatcher.anyOf((String) charsOrNone) : LATIN1_WHITESPACE; + return stringLStrip(self, matcher); + } + + @StarlarkMethod( + name = "rstrip", + doc = + "Returns a copy of the string where trailing characters that appear in " + + "chars are removed. Note that chars " + + "is not a suffix: all combinations of its value are removed:" + + "
"
+              + "\"abcbaa\".rstrip(\"ab\") == \"abc\""
+              + "
", + parameters = { + @Param(name = "self", doc = "This string."), + @Param( + name = "chars", + allowedTypes = { + @ParamType(type = String.class), + @ParamType(type = NoneType.class), + }, + doc = "The characters to remove, or all whitespace if None.", + defaultValue = "None") + }) + public String rstrip(String self, Object charsOrNone) { + CharMatcher matcher = + charsOrNone != Starlark.NONE ? CharMatcher.anyOf((String) charsOrNone) : LATIN1_WHITESPACE; + return stringRStrip(self, matcher); + } + + @StarlarkMethod( + name = "strip", + doc = + "Returns a copy of the string where leading or trailing characters that appear in " + + "chars are removed. Note that chars " + + "is neither a prefix nor a suffix: all combinations of its value " + + "are removed:" + + "
"
+              + "\"aabcbcbaa\".strip(\"ab\") == \"cbc\""
+              + "
", + parameters = { + @Param(name = "self", doc = "This string."), + @Param( + name = "chars", + allowedTypes = { + @ParamType(type = String.class), + @ParamType(type = NoneType.class), + }, + doc = "The characters to remove, or all whitespace if None.", + defaultValue = "None") + }) + public String strip(String self, Object charsOrNone) { + CharMatcher matcher = + charsOrNone != Starlark.NONE ? CharMatcher.anyOf((String) charsOrNone) : LATIN1_WHITESPACE; + return stringStrip(self, matcher); + } + + @StarlarkMethod( + name = "replace", + doc = + "Returns a copy of the string in which the occurrences " + + "of old have been replaced with new, optionally " + + "restricting the number of replacements to count.", + parameters = { + @Param(name = "self", doc = "This string."), + @Param(name = "old", doc = "The string to be replaced."), + @Param(name = "new", doc = "The string to replace with."), + @Param( + name = "count", + defaultValue = "-1", + doc = + "The maximum number of replacements. If omitted, or if the value is negative, " + + "there is no limit.") + }, + useStarlarkThread = true) + public String replace( + String self, String oldString, String newString, StarlarkInt countI, StarlarkThread thread) + throws EvalException { + int count = countI.toInt("count"); + if (count < 0) { + count = Integer.MAX_VALUE; + } + + StringBuilder sb = new StringBuilder(); + int start = 0; + for (int i = 0; i < count; i++) { + if (oldString.isEmpty()) { + sb.append(newString); + if (start < self.length()) { + sb.append(self.charAt(start++)); + } else { + break; + } + } else { + int end = self.indexOf(oldString, start); + if (end < 0) { + break; + } + sb.append(self, start, end).append(newString); + start = end + oldString.length(); + } + } + sb.append(self, start, self.length()); + return sb.toString(); + } + + @StarlarkMethod( + name = "split", + doc = + "Returns a list of all the words in the string, using sep as the " + + "separator, optionally limiting the number of splits to maxsplit.", + parameters = { + @Param(name = "self", doc = "This string."), + @Param(name = "sep", doc = "The string to split on."), + @Param( + name = "maxsplit", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "None", + doc = "The maximum number of splits.") + }, + useStarlarkThread = true) + public StarlarkList split( + String self, String sep, Object maxSplitO, StarlarkThread thread) throws EvalException { + if (sep.isEmpty()) { + throw Starlark.errorf("Empty separator"); + } + int maxSplit = Integer.MAX_VALUE; + if (maxSplitO != Starlark.NONE) { + maxSplit = Starlark.toInt(maxSplitO, "maxsplit"); + } + StarlarkList res = StarlarkList.newList(thread.mutability()); + int start = 0; + while (true) { + int end = self.indexOf(sep, start); + if (end < 0 || maxSplit-- == 0) { + res.addElement(self.substring(start)); + break; + } + res.addElement(self.substring(start, end)); + start = end + sep.length(); + } + return res; + } + + @StarlarkMethod( + name = "rsplit", + doc = + "Returns a list of all the words in the string, using sep as the " + + "separator, optionally limiting the number of splits to maxsplit. " + + "Except for splitting from the right, this method behaves like split().", + parameters = { + @Param(name = "self", doc = "This string."), + @Param(name = "sep", doc = "The string to split on."), + @Param( + name = "maxsplit", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "None", + doc = "The maximum number of splits.") + }, + useStarlarkThread = true) + public StarlarkList rsplit( + String self, String sep, Object maxSplitO, StarlarkThread thread) throws EvalException { + if (sep.isEmpty()) { + throw Starlark.errorf("Empty separator"); + } + int maxSplit = Integer.MAX_VALUE; + if (maxSplitO != Starlark.NONE) { + maxSplit = Starlark.toInt(maxSplitO, "maxsplit"); + } + ArrayList res = new ArrayList<>(); + int end = self.length(); + while (true) { + int start = self.lastIndexOf(sep, end - 1); + if (start < 0 || maxSplit-- == 0) { + res.add(self.substring(0, end)); + break; + } + res.add(self.substring(start + sep.length(), end)); + end = start; + } + Collections.reverse(res); + return StarlarkList.copyOf(thread.mutability(), res); + } + + @StarlarkMethod( + name = "partition", + doc = + "Splits the input string at the first occurrence of the separator sep and" + + " returns the resulting partition as a three-element tuple of the form (before," + + " separator, after). If the input string does not contain the separator, partition" + + " returns (self, '', '').", + parameters = {@Param(name = "self"), @Param(name = "sep", doc = "The string to split on.")}) + public Tuple partition(String self, String sep) throws EvalException { + return partitionCommon(self, sep, /*first=*/ true); + } + + @StarlarkMethod( + name = "rpartition", + doc = + "Splits the input string at the last occurrence of the separator sep and" + + " returns the resulting partition as a three-element tuple of the form (before," + + " separator, after). If the input string does not contain the separator," + + " rpartition returns ('', '', self).", + parameters = {@Param(name = "self"), @Param(name = "sep", doc = "The string to split on.")}) + public Tuple rpartition(String self, String sep) throws EvalException { + return partitionCommon(self, sep, /*first=*/ false); + } + + // Splits input at the first or last occurrence of the given separator, + // and returns a triple of substrings (before, separator, after). + // If the input does not contain the separator, + // it returns (input, "", "") if first, or ("", "", input), if !first. + private static Tuple partitionCommon(String input, String separator, boolean first) + throws EvalException { + if (separator.isEmpty()) { + throw Starlark.errorf("empty separator"); + } + + String a = ""; + String b = ""; + String c = ""; + + int pos = first ? input.indexOf(separator) : input.lastIndexOf(separator); + if (pos < 0) { + if (first) { + a = input; + } else { + c = input; + } + } else { + a = input.substring(0, pos); + b = separator; + c = input.substring(pos + separator.length()); + } + + return Tuple.triple(a, b, c); + } + + @StarlarkMethod( + name = "capitalize", + doc = + "Returns a copy of the string with its first character (if any) capitalized and the rest " + + "lowercased. This method does not support non-ascii characters. ", + parameters = {@Param(name = "self", doc = "This string.")}) + public String capitalize(String self) throws EvalException { + if (self.isEmpty()) { + return self; + } + // TODO(adonovan): fix: support non-ASCII characters. Requires that Bazel stop abusing Latin1. + return Character.toUpperCase(self.charAt(0)) + Ascii.toLowerCase(self.substring(1)); + } + + @StarlarkMethod( + name = "title", + doc = + "Converts the input string into title case, i.e. every word starts with an " + + "uppercase letter while the remaining letters are lowercase. In this " + + "context, a word means strictly a sequence of letters. This method does " + + "not support supplementary Unicode characters.", + parameters = {@Param(name = "self", doc = "This string.")}) + public String title(String self) throws EvalException { + char[] data = self.toCharArray(); + boolean previousWasLetter = false; + + for (int pos = 0; pos < data.length; ++pos) { + char current = data[pos]; + boolean currentIsLetter = Character.isLetter(current); + + if (currentIsLetter) { + if (previousWasLetter && Character.isUpperCase(current)) { + data[pos] = Character.toLowerCase(current); + } else if (!previousWasLetter && Character.isLowerCase(current)) { + data[pos] = Character.toUpperCase(current); + } + } + previousWasLetter = currentIsLetter; + } + + return new String(data); + } + + /** + * Common implementation for find, rfind, index, rindex. + * + * @param forward true if we want to return the last matching index. + */ + private static int stringFind(boolean forward, String self, String sub, Object start, Object end) + throws EvalException { + long indices = substringIndices(self, start, end); + int startpos = lo(indices); + int endpos = hi(indices); + if (forward) { + return self.indexOf(sub, startpos, endpos); + } + // String#lastIndexOf can't be used to implement rfind() because it only + // confines the start position of the substring, not the entire substring. + int subpos = self.substring(startpos, endpos).lastIndexOf(sub); + return subpos < 0 + ? subpos // + : subpos + startpos; + } + + private static final Pattern SPLIT_LINES_PATTERN = + Pattern.compile("(?.*)(?(\\r\\n|\\r|\\n)?)"); + + @StarlarkMethod( + name = "rfind", + doc = + "Returns the last index where sub is found, or -1 if no such index exists, " + + "optionally restricting to [start:end], " + + "start being inclusive and end being exclusive.", + parameters = { + @Param(name = "self", doc = "This string."), + @Param(name = "sub", doc = "The substring to find."), + @Param( + name = "start", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "0", + doc = "Restrict to search from this position."), + @Param( + name = "end", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "None", + doc = "optional position before which to restrict to search.") + }) + public int rfind(String self, String sub, Object start, Object end) throws EvalException { + return stringFind(false, self, sub, start, end); + } + + @StarlarkMethod( + name = "find", + doc = + "Returns the first index where sub is found, or -1 if no such index exists, " + + "optionally restricting to [start:end], " + + "start being inclusive and end being exclusive.", + parameters = { + @Param(name = "self", doc = "This string."), + @Param(name = "sub", doc = "The substring to find."), + @Param( + name = "start", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "0", + doc = "Restrict to search from this position."), + @Param( + name = "end", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "None", + doc = "optional position before which to restrict to search.") + }) + public int find(String self, String sub, Object start, Object end) throws EvalException { + return stringFind(true, self, sub, start, end); + } + + @StarlarkMethod( + name = "rindex", + doc = + "Returns the last index where sub is found, or raises an error if no such " + + "index exists, optionally restricting to [start:end], " + + "start being inclusive and end being exclusive.", + parameters = { + @Param(name = "self", doc = "This string."), + @Param(name = "sub", doc = "The substring to find."), + @Param( + name = "start", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "0", + doc = "Restrict to search from this position."), + @Param( + name = "end", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "None", + doc = "optional position before which to restrict to search.") + }) + public int rindex(String self, String sub, Object start, Object end) throws EvalException { + int res = stringFind(false, self, sub, start, end); + if (res < 0) { + throw Starlark.errorf("substring not found"); + } + return res; + } + + @StarlarkMethod( + name = "index", + doc = + "Returns the first index where sub is found, or raises an error if no such " + + " index exists, optionally restricting to [start:end]" + + "start being inclusive and end being exclusive.", + parameters = { + @Param(name = "self", doc = "This string."), + @Param(name = "sub", doc = "The substring to find."), + @Param( + name = "start", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "0", + doc = "Restrict to search from this position."), + @Param( + name = "end", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "None", + doc = "optional position before which to restrict to search.") + }) + public int index(String self, String sub, Object start, Object end) throws EvalException { + int res = stringFind(true, self, sub, start, end); + if (res < 0) { + throw Starlark.errorf("substring not found"); + } + return res; + } + + @StarlarkMethod( + name = "splitlines", + doc = + "Splits the string at line boundaries ('\\n', '\\r\\n', '\\r') " + + "and returns the result as a new mutable list.", + parameters = { + @Param(name = "self", doc = "This string."), + @Param( + // TODO(b/67740837): clarify whether this is named or positional. + name = "keepends", + defaultValue = "False", + doc = "Whether the line breaks should be included in the resulting list.") + }, + useStarlarkThread = true) + public Sequence splitLines(String self, boolean keepEnds, StarlarkThread thread) + throws EvalException { + StarlarkList result = StarlarkList.newList(thread.mutability()); + Matcher matcher = SPLIT_LINES_PATTERN.matcher(self); + while (matcher.find()) { + String line = matcher.group("line"); + String lineBreak = matcher.group("break"); + boolean trailingBreak = lineBreak.isEmpty(); + if (line.isEmpty() && trailingBreak) { + break; + } + if (keepEnds && !trailingBreak) { + result.addElement(line + lineBreak); + } else { + result.addElement(line); + } + } + // TODO(adonovan): spec should state that result is mutable, + // as in Python[23] and go.starlark.net. + return result; + } + + @StarlarkMethod( + name = "isalpha", + doc = + "Returns True if all characters in the string are alphabetic ([a-zA-Z]) and there is " + + "at least one character.", + parameters = {@Param(name = "self", doc = "This string.")}) + public boolean isAlpha(String self) throws EvalException { + return matches(self, ALPHA, false); + } + + @StarlarkMethod( + name = "isalnum", + doc = + "Returns True if all characters in the string are alphanumeric ([a-zA-Z0-9]) and there " + + "is at least one character.", + parameters = {@Param(name = "self", doc = "This string.")}) + public boolean isAlnum(String self) throws EvalException { + return matches(self, ALNUM, false); + } + + @StarlarkMethod( + name = "isdigit", + doc = + "Returns True if all characters in the string are digits ([0-9]) and there is " + + "at least one character.", + parameters = {@Param(name = "self", doc = "This string.")}) + public boolean isDigit(String self) throws EvalException { + return matches(self, DIGIT, false); + } + + @StarlarkMethod( + name = "isspace", + doc = + "Returns True if all characters are white space characters and the string " + + "contains at least one character.", + parameters = {@Param(name = "self", doc = "This string.")}) + public boolean isSpace(String self) throws EvalException { + return matches(self, SPACE, false); + } + + @StarlarkMethod( + name = "islower", + doc = + "Returns True if all cased characters in the string are lowercase and there is " + + "at least one character.", + parameters = {@Param(name = "self", doc = "This string.")}) + public boolean isLower(String self) throws EvalException { + // Python also accepts non-cased characters, so we cannot use LOWER. + return matches(self, UPPER.negate(), true); + } + + @StarlarkMethod( + name = "isupper", + doc = + "Returns True if all cased characters in the string are uppercase and there is " + + "at least one character.", + parameters = {@Param(name = "self", doc = "This string.")}) + public boolean isUpper(String self) throws EvalException { + // Python also accepts non-cased characters, so we cannot use UPPER. + return matches(self, LOWER.negate(), true); + } + + @StarlarkMethod( + name = "istitle", + doc = + "Returns True if the string is in title case and it contains at least one character. " + + "This means that every uppercase character must follow an uncased one (e.g. " + + "whitespace) and every lowercase character must follow a cased one (e.g. " + + "uppercase or lowercase).", + parameters = {@Param(name = "self", doc = "This string.")}) + public boolean isTitle(String self) throws EvalException { + if (self.isEmpty()) { + return false; + } + // From the Python documentation: "uppercase characters may only follow uncased characters + // and lowercase characters only cased ones". + char[] data = self.toCharArray(); + CharMatcher matcher = CharMatcher.any(); + char leftMostCased = ' '; + for (int pos = data.length - 1; pos >= 0; --pos) { + char current = data[pos]; + // 1. Check condition that was determined by the right neighbor. + if (!matcher.matches(current)) { + return false; + } + // 2. Determine condition for the left neighbor. + if (LOWER.matches(current)) { + matcher = CASED; + } else if (UPPER.matches(current)) { + matcher = CASED.negate(); + } else { + matcher = CharMatcher.any(); + } + // 3. Store character if it is cased. + if (CASED.matches(current)) { + leftMostCased = current; + } + } + // The leftmost cased letter must be uppercase. If leftMostCased is not a cased letter here, + // then the string doesn't have any cased letter, so UPPER.test will return false. + return UPPER.matches(leftMostCased); + } + + private static boolean matches( + String str, CharMatcher matcher, boolean requiresAtLeastOneCasedLetter) { + if (str.isEmpty()) { + return false; + } else if (!requiresAtLeastOneCasedLetter) { + return matcher.matchesAllOf(str); + } + int casedLetters = 0; + for (char current : str.toCharArray()) { + if (!matcher.matches(current)) { + return false; + } else if (requiresAtLeastOneCasedLetter && CASED.matches(current)) { + ++casedLetters; + } + } + return casedLetters > 0; + } + + private static final CharMatcher DIGIT = CharMatcher.javaDigit(); + private static final CharMatcher LOWER = CharMatcher.inRange('a', 'z'); + private static final CharMatcher UPPER = CharMatcher.inRange('A', 'Z'); + private static final CharMatcher ALPHA = LOWER.or(UPPER); + private static final CharMatcher ALNUM = ALPHA.or(DIGIT); + private static final CharMatcher CASED = ALPHA; + private static final CharMatcher SPACE = CharMatcher.whitespace(); + + @StarlarkMethod( + name = "count", + doc = + "Returns the number of (non-overlapping) occurrences of substring sub in " + + "string, optionally restricting to [start:end], start " + + "being inclusive and end being exclusive.", + parameters = { + @Param(name = "self", doc = "This string."), + @Param(name = "sub", doc = "The substring to count."), + @Param( + name = "start", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "0", + doc = "Restrict to search from this position."), + @Param( + name = "end", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "None", + doc = "optional position before which to restrict to search.") + }) + public int count(String self, String sub, Object start, Object end) throws EvalException { + long indices = substringIndices(self, start, end); + if (sub.isEmpty()) { + return hi(indices) - lo(indices) + 1; // str.length() + 1 + } + // Unfortunately Java forces us to allocate here, even though + // String has a private indexOf method that accepts indices. + // Fortunately the common case is self[0:n]. + String str = self.substring(lo(indices), hi(indices)); + int count = 0; + int index = 0; + while ((index = str.indexOf(sub, index)) >= 0) { + count++; + index += sub.length(); + } + return count; + } + + @StarlarkMethod( + name = "elems", + doc = + "Returns an iterable value containing successive 1-element substrings of the string. " + + "Equivalent to [s[i] for i in range(len(s))], except that the " + + "returned value might not be a list.", + parameters = {@Param(name = "self", doc = "This string.")}) + public Sequence elems(String self) { + // TODO(adonovan): opt: return a new type that is lazily iterable. + char[] chars = self.toCharArray(); + Object[] strings = new Object[chars.length]; + for (int i = 0; i < chars.length; i++) { + strings[i] = memoizedCharToString(chars[i]); + } + return StarlarkList.wrap(null, strings); + } + + @StarlarkMethod( + name = "endswith", + doc = + "Returns True if the string ends with sub, otherwise False, optionally " + + "restricting to [start:end], start being inclusive " + + "and end being exclusive.", + parameters = { + @Param(name = "self", doc = "This string."), + @Param( + name = "sub", + allowedTypes = { + @ParamType(type = String.class), + @ParamType(type = Tuple.class, generic1 = String.class), + }, + doc = "The suffix (or tuple of alternative suffixes) to match."), + @Param( + name = "start", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "0", + doc = "Test beginning at this position."), + @Param( + name = "end", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "None", + doc = "optional position at which to stop comparing.") + }) + public boolean endsWith(String self, Object sub, Object start, Object end) throws EvalException { + long indices = substringIndices(self, start, end); + if (sub instanceof String) { + return substringEndsWith(self, lo(indices), hi(indices), (String) sub); + } + for (String s : Sequence.cast(sub, String.class, "sub")) { + if (substringEndsWith(self, lo(indices), hi(indices), s)) { + return true; + } + } + return false; + } + + // Computes str.substring(start, end).endsWith(suffix) without allocation. + private static boolean substringEndsWith(String str, int start, int end, String suffix) { + int n = suffix.length(); + return start + n <= end && str.regionMatches(end - n, suffix, 0, n); + } + + // In Python, formatting is very complex. + // We handle here the simplest case which provides most of the value of the function. + // https://docs.python.org/3/library/string.html#formatstrings + @StarlarkMethod( + name = "format", + doc = + "Perform string interpolation. Format strings contain replacement fields surrounded by" + + " curly braces {}. Anything that is not contained in braces" + + " is considered literal text, which is copied unchanged to the output.If you need" + + " to include a brace character in the literal text, it can be escaped by doubling:" + + " {{ and }}A replacement field can be" + + " either a name, a number, or empty. Values are converted to strings using the str function.# Access in order:\n" + + "\"{} < {}\".format(4, 5) == \"4 < 5\"\n" + + "# Access by position:\n" + + "\"{1}, {0}\".format(2, 1) == \"1, 2\"\n" + + "# Access by name:\n" + + "\"x{key}x\".format(key = 2) == \"x2x\"\n", + parameters = { + @Param(name = "self", doc = "This string."), + }, + extraPositionals = @Param(name = "args", defaultValue = "()", doc = "List of arguments."), + extraKeywords = + @Param(name = "kwargs", defaultValue = "{}", doc = "Dictionary of arguments."), + useStarlarkThread = true) + public String format(String self, Tuple args, Dict kwargs, StarlarkThread thread) + throws EvalException { + return new FormatParser().format(self, args, kwargs, thread.getSemantics()); + } + + @StarlarkMethod( + name = "startswith", + doc = + "Returns True if the string starts with sub, otherwise False, optionally " + + "restricting to [start:end], start being inclusive and " + + "end being exclusive.", + parameters = { + @Param(name = "self", doc = "This string."), + @Param( + name = "sub", + allowedTypes = { + @ParamType(type = String.class), + @ParamType(type = Tuple.class, generic1 = String.class), + }, + doc = "The prefix (or tuple of alternative prefixes) to match."), + @Param( + name = "start", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "0", + doc = "Test beginning at this position."), + @Param( + name = "end", + allowedTypes = { + @ParamType(type = StarlarkInt.class), + @ParamType(type = NoneType.class), + }, + defaultValue = "None", + doc = "Stop comparing at this position.") + }) + public boolean startsWith(String self, Object sub, Object start, Object end) + throws EvalException { + long indices = substringIndices(self, start, end); + if (sub instanceof String) { + return substringStartsWith(self, lo(indices), hi(indices), (String) sub); + } + for (String s : Sequence.cast(sub, String.class, "sub")) { + if (substringStartsWith(self, lo(indices), hi(indices), s)) { + return true; + } + } + return false; + } + + // Computes str.substring(start, end).startsWith(prefix) without allocation. + private static boolean substringStartsWith(String str, int start, int end, String prefix) { + return start + prefix.length() <= end && str.startsWith(prefix, start); + } + + @StarlarkMethod( + name = "removeprefix", + doc = + "If the string starts with prefix, returns a new string with the prefix " + + "removed. Otherwise, returns the string.", + parameters = { + @Param(name = "self", doc = "This string."), + @Param(name = "prefix", doc = "The prefix to remove if present."), + }) + public String removePrefix(String self, String prefix) { + if (self.startsWith(prefix)) { + return self.substring(prefix.length()); + } + return self; + } + + @StarlarkMethod( + name = "removesuffix", + doc = + "If the string ends with suffix, returns a new string with the suffix " + + "removed. Otherwise, returns the string.", + parameters = { + @Param(name = "self", doc = "This string."), + @Param(name = "suffix", doc = "The suffix to remove if present."), + }) + public String removeSuffix(String self, String suffix) { + if (self.endsWith(suffix)) { + return self.substring(0, self.length() - suffix.length()); + } + return self; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Structure.java b/third_party/bazel/main/java/net/starlark/java/eval/Structure.java new file mode 100644 index 000000000..3487945fa --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/Structure.java @@ -0,0 +1,82 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.collect.ImmutableCollection; +import javax.annotation.Nullable; + +/** + * An interface for Starlark values (such as Bazel structs) with fields that may be accessed using + * Starlark's {@code x.field} notation and optionally updating using an {@code x.f=y} assignment. + */ +public interface Structure extends StarlarkValue { + + /** + * Returns the value of the field with the given name, or null if the field does not exist. The + * interpreter (Starlark code) calls the getValue below, which has access to StarlarkSemantics. + * + *

The set of names for which {@code getValue} returns non-null should match {@code + * getFieldNames} if possible. + * + * @throws EvalException if a user-visible error occurs (other than non-existent field). + */ + // TODO(adonovan): rename "getField". + @Nullable + Object getValue(String name) throws EvalException; + + /** + * Returns the value of the field with the given name, or null if the field does not exist. The + * interpreter (Starlark code) calls this getValue, but client code cannot be relied upon to do + * so, so any checks done on the semantics are incompletely enforced. + * + * @param semantics the Starlark semantics, which determine the available fields + * @param name the name of the field to retrieve + * @throws EvalException if the field exists but could not be retrieved + */ + @Nullable + default Object getValue(StarlarkSemantics semantics, String name) throws EvalException { + return this.getValue(name); + } + + /** + * Returns the names of this value's fields, in some undefined but stable order. + * + *

A call to {@code getValue} for each of these names should return non-null, though this is + * not enforced. + * + *

The Starlark expression {@code dir(x)} reports the union of {@code getFieldNames()} and any + * StarlarkMethod-annotated fields and methods of this value. + */ + ImmutableCollection getFieldNames(); + + /** + * Returns the error message to print for an attempt to access an undefined field. + * + *

May return null to use a default error message. + */ + @Nullable + String getErrorMessageForUnknownField(String field); + + /** + * Updates the named field of this value as if by the Starlark statement {@code this.field = + * value}. + * + * @throws EvalException if the update failed because this value is immutable, does not support + * field update, or update of that particular field, or because the value was inappropriate. + */ + default void setField(String field, Object value) throws EvalException { + throw Starlark.errorf("%s value does not support field assignment", Starlark.type(this)); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/SymbolGenerator.java b/third_party/bazel/main/java/net/starlark/java/eval/SymbolGenerator.java new file mode 100644 index 000000000..e2525f863 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/SymbolGenerator.java @@ -0,0 +1,138 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.eval; + +import com.google.auto.value.AutoValue; + +/** + * Class to be used when an object wants to be compared using reference equality. Since reference + * equality is not usable when comparing objects across multiple Starlark evaluations, we use a more + * stable method: an object identifying the {@link #owner} of the current Starlark context, and an + * {@link #index} indicating how many reference-equal objects have already been created (and + * therefore asked for a unique symbol for themselves). Global symbols may also be identified using + * their exported name rather than an anonymous index. + * + *

Objects that want to use reference equality should instead call {@link #generate} on a + * provided {@code SymbolGenerator} instance, and compare the returned object for equality, since it + * will be stable across identical Starlark evaluations. Note that equality comparisons are + * invalidated by any change to the inputs of a Starlark evaluation. For example, it is not valid to + * compare two values that came from different Bazel builds with an intervening edit to a .bzl file. + * + *

For Starlark values that rely on this class, equality comparison across Starlark threads is + * not guaranteed to be consistent until both threads are done running. This is due to the edge case + * of one value being exported while the other is still unexported, since the export process can + * change the equality token. + */ +public final class SymbolGenerator { + private final T owner; + private int index = 0; + + /** + * Creates a new symbol generator for the Starlark evaluation uniquely identified by the given + * owner. + * + *

Precisely, two {@code SymbolGenerators} that have owners {@code o1} and {@code o2} are + * considered to be for the same Starlark evaluation, if and only if {@code o1.equals(o2)}. + */ + public static SymbolGenerator create(T owner) { + return new SymbolGenerator<>(owner); + } + + /** + * Creates a generator for a Starlark evaluation whose values don't require strict reference + * equality checks. + * + *

This can be used in the following cases. + * + *

    + *
  • The result of a Starlark evaluation has a simple type (like numbers or strings) where + * values are compared, not object references. + *
  • The result is temporary and it won't be stored, transmitted or regenerated while being + * retained. + *
+ * + *

The "regenerated while being retained" condition may occur, for example, if a part of the + * resulting value is retained somewhere in the process, but the value itself is evicted from a + * cache and is subsequently regenerated. + */ + public static SymbolGenerator createTransient() { + return create(new Object()); + } + + private SymbolGenerator(T owner) { + this.owner = owner; + } + + public synchronized Symbol generate() { + return LocalSymbol.create(owner, index++); + } + + T getOwner() { + return owner; + } + + /** Identifier for an object created by a uniquely defined Starlark thread. */ + // TODO(bazel-team): The name "Symbol", in the context of an interpreter, is a bit confusing. + // Consider renaming to "Token" or similar. + public abstract static class Symbol { + /** + * Creates a new {@link GlobalSymbol} with the same owner as this symbol. + * + *

Objects may start with a {@link LocalSymbol} and are later exported with a global name. + * This method can be used to create a suitable {@link GlobalSymbol}. + */ + public final GlobalSymbol exportAs(String name) { + return GlobalSymbol.create(getOwner(), name); + } + + public abstract T getOwner(); + + public abstract boolean isGlobal(); + } + + @AutoValue + abstract static class LocalSymbol extends Symbol { + private static LocalSymbol create(T owner, int index) { + return new AutoValue_SymbolGenerator_LocalSymbol<>(owner, index); + } + + abstract int getIndex(); + + @Override + public final boolean isGlobal() { + return false; + } + } + + /** + * An identifier for a global variable. + * + *

Intended as an optimization, allowing the lookup of a global variable from its GlobalSymbol, + * e.g. for deserialization: the owner should be a wrapper object for a {@link Module}, and we can + * obtain the value from the symbol's name and {@link Module#getGlobal}. + */ + @AutoValue + public abstract static class GlobalSymbol extends Symbol { + private static GlobalSymbol create(T owner, String name) { + return new AutoValue_SymbolGenerator_GlobalSymbol<>(owner, name); + } + + public abstract String getName(); + + @Override + public final boolean isGlobal() { + return true; + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Tuple.java b/third_party/bazel/main/java/net/starlark/java/eval/Tuple.java new file mode 100644 index 000000000..4aa679ed0 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/Tuple.java @@ -0,0 +1,161 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.eval; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.ObjectArrays; +import java.util.AbstractCollection; +import java.util.AbstractList; +import java.util.Arrays; +import java.util.Iterator; +import net.starlark.java.annot.StarlarkBuiltin; + +/** A Tuple is an immutable finite sequence of values. */ +@StarlarkBuiltin( + name = "tuple", + category = "core", + doc = + "The built-in tuple type. Example tuple expressions:
" + + "

x = (1, 2, 3)
" + + "Accessing elements is possible using indexing (starts from 0):
" + + "
e = x[1]   # e == 2
" + + "Lists support the + operator to concatenate two tuples. Example:
" + + "
x = (1, 2) + (3, 4)   # x == (1, 2, 3, 4)\n"
+            + "x = (\"a\", \"b\")\n"
+            + "x += (\"c\",)            # x == (\"a\", \"b\", \"c\")
" + + "Similar to lists, tuples support slice operations:" + + "
('a', 'b', 'c', 'd')[1:3]   # ('b', 'c')\n"
+            + "('a', 'b', 'c', 'd')[::2]  # ('a', 'c')\n"
+            + "('a', 'b', 'c', 'd')[3:0:-1]  # ('d', 'c', 'b')
" + + "Tuples are immutable, therefore x[1] = \"a\" is not supported.") +public abstract class Tuple extends AbstractList + implements Sequence, Comparable { + + // Prohibit instantiation outside of package. + Tuple() {} + + /** Returns the empty tuple. */ + public static Tuple empty() { + return RegularTuple.EMPTY; + } + + /** Returns a Tuple that wraps the specified array, which must not be subsequently modified. */ + static Tuple wrap(Object[] array) { + switch (array.length) { + case 0: + return RegularTuple.EMPTY; + case 1: + return new SingletonTuple(array[0]); + default: + return new RegularTuple(array); + } + } + + /** Returns a tuple containing the given elements. */ + public static Tuple copyOf(Iterable seq) { + if (seq instanceof Tuple) { + return (Tuple) seq; + } + return wrap(Iterables.toArray(seq, Object.class)); + } + + /** Returns a tuple containing the given elements. */ + public static Tuple of(Object... elems) { + return wrap(Arrays.copyOf(elems, elems.length)); + } + + /** Returns a two-element tuple. */ + public static Tuple pair(Object a, Object b) { + // Equivalent to of(a, b) but avoids variadic array allocation. + return wrap(new Object[] {a, b}); + } + + /** Returns a three-element tuple. */ + public static Tuple triple(Object a, Object b, Object c) { + // Equivalent to of(a, b, c) but avoids variadic array allocation. + return wrap(new Object[] {a, b, c}); + } + + /** Returns a tuple that is the concatenation of two tuples. */ + public static Tuple concat(Tuple x, Tuple y) { + if (x.isEmpty()) { + return y; + } else if (y.isEmpty()) { + return x; + } else { + Object[] xelems = + x instanceof SingletonTuple + ? new Object[] {((SingletonTuple) x).elem} + : ((RegularTuple) x).elems; + Object[] yelems = + y instanceof SingletonTuple + ? new Object[] {((SingletonTuple) y).elem} + : ((RegularTuple) y).elems; + return wrap(ObjectArrays.concat(xelems, yelems, Object.class)); + } + } + + @Override + public int compareTo(Tuple that) { + return Sequence.compare(this, that); + } + + @Override + public boolean equals(Object that) { + // This slightly violates the java.util.List equivalence contract + // because it considers the class, not just the elements. + // This is needed because in Starlark tuples are never equal to lists, however in Java they both + // implement List interface. + return this == that || (that instanceof Tuple && Sequence.sameElems(this, ((Tuple) that))); + } + + // TODO(adonovan): StarlarkValue has 3 String methods yet still we need this fourth. Why? + @Override + public String toString() { + return Starlark.repr(this); + } + + /** + * Returns a new ImmutableList backed by {@code array}, which must not be subsequently + * modified. + */ + // TODO(adonovan): move this somewhere more appropriate. + static ImmutableList wrapImmutable(Object[] array) { + // Construct an ImmutableList that shares the array. + // ImmutableList relies on the implementation of Collection.toArray + // not subsequently modifying the returned array. + return ImmutableList.copyOf( + new AbstractCollection() { + @Override + public Object[] toArray() { + return array; + } + + @Override + public int size() { + return array.length; + } + + @Override + public Iterator iterator() { + throw new UnsupportedOperationException(); + } + }); + } + + /** Returns a Tuple containing n consecutive repeats of this tuple. */ + abstract Tuple repeat(StarlarkInt n) throws EvalException; +} diff --git a/third_party/bazel/main/java/net/starlark/java/eval/cpu_profiler_posix.cc b/third_party/bazel/main/java/net/starlark/java/eval/cpu_profiler_posix.cc new file mode 100644 index 000000000..a88ea850a --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/cpu_profiler_posix.cc @@ -0,0 +1,203 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// POSIX support for Starlark CPU profiler. + +#include // for htonl +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __linux__ +#include +#else // darwin +#include +#endif + +namespace cpu_profiler { + +// static native boolean supported(); +extern "C" JNIEXPORT jboolean JNICALL +Java_net_starlark_java_eval_CpuProfiler_supported(JNIEnv *env, jclass clazz) { + return true; +} + +static int fd; // the write end of the profile event pipe + +pid_t gettid(void) { +#ifdef __linux__ + return (pid_t)syscall(SYS_gettid); +#else // darwin + uint64_t tid64; + pthread_threadid_np(NULL, &tid64); + return (pid_t)tid64; +#endif +} + +// SIGPROF handler. +// Warning: asynchronous! See signal-safety(7) for the programming discipline. +void onsigprof(int sig) { + int old_errno = errno; + + if (fd == 0) { + const char *msg = "startTimer called before createPipe\n"; + write(2, msg, strlen(msg)); + abort(); + } + + // Send an event containing the int32be-encoded OS thread ID. + pid_t tid = gettid(); + uint32_t tid_be = htonl(tid); + int r = write(fd, (void *)&tid_be, sizeof tid_be); + if (r < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + // The Java router thread cannot keep up. + // + // A busy 12-core machine receives 12 * 100Hz = 1200 signals per second, + // and thus writes 4.8KB/s to the pipe. The default pipe buffer + // size on Linux is 64KiB, sufficient to buffer ~14s of data. + // (It is a quarter of that on Mac OS X.) + // + // Rather than block in write(2), causing the JVM to deadlock, + // we print an error and discard the event. + const char *msg = + "Starlark profile event router thread cannot keep up; discarding " + "events\n"; + write(2, msg, strlen(msg)); + } else { + // We shouldn't use perror in a signal handler. + // Strictly, we shouldn't use strerror either, + // but for all errors returned by write it merely + // returns a constant. + char buf[1024] = "write: "; + strncat(buf, strerror(errno), sizeof buf - strlen(buf) - 1); + strncat(buf, "\n", sizeof buf - strlen(buf) - 1); + write(2, buf, strlen(buf)); + abort(); + } + } + + errno = old_errno; +} + +// static native jint gettid(); +extern "C" JNIEXPORT jint JNICALL +Java_net_starlark_java_eval_CpuProfiler_gettid(JNIEnv *env, jclass clazz) { + return gettid(); +} + +// makeFD: return new FileDescriptor(fd) +// +// This would be easy to do in Java, but for the field being private. +// Java really does everything it can to make system programming hateful. +static jobject makeFD(JNIEnv *env, int fd) { + jclass fdclass = env->FindClass("java/io/FileDescriptor"); + if (fdclass == nullptr) return nullptr; // exception + + jmethodID init = env->GetMethodID(fdclass, "", "()V"); + if (init == nullptr) return nullptr; // exception + jobject fdobj = env->NewObject(fdclass, init); + + jfieldID fd_field = env->GetFieldID(fdclass, "fd", "I"); + if (fd_field == nullptr) return nullptr; // exception + env->SetIntField(fdobj, fd_field, fd); + + return fdobj; +} + +// static native FileDescriptor createPipe(); +extern "C" JNIEXPORT jobject JNICALL +Java_net_starlark_java_eval_CpuProfiler_createPipe(JNIEnv *env, jclass clazz) { + // Create a pipe for profile events from the handler to Java. + // The default pipe size is 64KiB on Linux and 16KiB on Mac OS X. + int pipefds[2]; + if (pipe(pipefds) < 0) { + perror("pipe"); + abort(); + } + fd = pipefds[1]; + + // Make the write end non-blocking so that the signal + // handler can detect overflow (rather than deadlock). + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK); + + // Return the read end of the event pipe, + // wrapped by a java FileDescriptor. + return makeFD(env, pipefds[0]); +} + +// static native boolean startTimer(long period_micros); +extern "C" JNIEXPORT jboolean JNICALL +Java_net_starlark_java_eval_CpuProfiler_startTimer(JNIEnv *env, jclass clazz, + jlong period_micros) { + // Install the signal handler. + // Use sigaction(2) not signal(2) so that we can correctly + // restore the previous handler if necessary. + struct sigaction oldact = {}, act = {}; + act.sa_handler = onsigprof; + act.sa_flags = SA_RESTART; // the JVM doesn't expect EINTR + if (sigaction(SIGPROF, &act, &oldact) < 0) { + perror("sigaction"); + abort(); + } + + // Is a handler already in effect? + // Check for 3-arg and 1-arg forms. + typedef void (*sighandler_t)(int); // don't rely on this GNU extension + sighandler_t prev = (oldact.sa_flags & SA_SIGINFO) != 0 + ? reinterpret_cast(oldact.sa_sigaction) + : oldact.sa_handler; + // The initial handler (DFL or IGN) may vary by thread package. + if (prev != SIG_DFL && prev != SIG_IGN) { + // Someone else is profiling this JVM. + // Restore their handler and fail. + (void)sigaction(SIGPROF, &oldact, nullptr); + return false; + } + + // Start the CPU interval timer. + struct timeval period = { + .tv_sec = 0, + .tv_usec = static_cast(period_micros), + }; + struct itimerval timer = {.it_interval = period, .it_value = period}; + if (setitimer(ITIMER_PROF, &timer, nullptr) < 0) { + perror("setitimer"); + abort(); + } + + return true; +} + +// static native void stopTimer(); +extern "C" JNIEXPORT void JNICALL +Java_net_starlark_java_eval_CpuProfiler_stopTimer(JNIEnv *env, jclass clazz) { + // Disarm the CPU interval timer. + struct itimerval timer = {}; + if (setitimer(ITIMER_PROF, &timer, nullptr) < 0) { + perror("setitimer"); + abort(); + } + + // Uninstall signal handler. + signal(SIGPROF, SIG_IGN); +} + +} // namespace cpu_profiler diff --git a/third_party/bazel/main/java/net/starlark/java/eval/cpu_profiler_unimpl.cc b/third_party/bazel/main/java/net/starlark/java/eval/cpu_profiler_unimpl.cc new file mode 100644 index 000000000..82d2ed393 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/eval/cpu_profiler_unimpl.cc @@ -0,0 +1,48 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Starlark CPU profiler stubs for unsupported platforms. + +#include +#include + +namespace cpu_profiler { + +extern "C" JNIEXPORT jboolean JNICALL +Java_net_starlark_java_eval_CpuProfiler_supported(JNIEnv *env, jclass clazz) { + return false; +} + +extern "C" JNIEXPORT jint JNICALL +Java_net_starlark_java_eval_CpuProfiler_gettid(JNIEnv *env, jclass clazz) { + abort(); +} + +extern "C" JNIEXPORT jobject JNICALL +Java_net_starlark_java_eval_CpuProfiler_createPipe(JNIEnv *env, jclass clazz) { + abort(); +} + +extern "C" JNIEXPORT jboolean JNICALL +Java_net_starlark_java_eval_CpuProfiler_startTimer(JNIEnv *env, jclass clazz, + jlong period_micros) { + abort(); +} + +extern "C" JNIEXPORT void JNICALL +Java_net_starlark_java_eval_CpuProfiler_stopTimer(JNIEnv *env, jclass clazz) { + abort(); +} + +} // namespace cpu_profiler diff --git a/third_party/bazel/main/java/net/starlark/java/lib/json/BUILD b/third_party/bazel/main/java/net/starlark/java/lib/json/BUILD new file mode 100644 index 000000000..0a6d435b4 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/lib/json/BUILD @@ -0,0 +1,24 @@ +load("@rules_java//java:defs.bzl", "java_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src:__subpackages__"], +) + +# Starlark json module +java_library( + name = "json", + srcs = ["Json.java"], + visibility = ["//src/main/java/net/starlark/java:clients"], + deps = [ + "//src/main/java/net/starlark/java/annot", + "//src/main/java/net/starlark/java/eval", + "//src/main/java/net/starlark/java/syntax", + ], +) diff --git a/third_party/bazel/main/java/net/starlark/java/lib/json/Json.java b/third_party/bazel/main/java/net/starlark/java/lib/json/Json.java new file mode 100644 index 000000000..a21338b6d --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/lib/json/Json.java @@ -0,0 +1,774 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.lib.json; + +import java.util.Arrays; +import java.util.Map; +import net.starlark.java.annot.Param; +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; +import net.starlark.java.eval.Dict; +import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.Mutability; +import net.starlark.java.eval.Starlark; +import net.starlark.java.eval.StarlarkFloat; +import net.starlark.java.eval.StarlarkInt; +import net.starlark.java.eval.StarlarkIterable; +import net.starlark.java.eval.StarlarkList; +import net.starlark.java.eval.StarlarkThread; +import net.starlark.java.eval.StarlarkValue; +import net.starlark.java.eval.Structure; + +// Tests at //src/test/java/net/starlark/java/eval:testdata/json.sky + +/** + * Json defines the Starlark {@code json} module, which provides functions for encoding/decoding + * Starlark values as JSON (https://tools.ietf.org/html/rfc8259). + */ +@StarlarkBuiltin( + name = "json", + category = "core.lib", + doc = "Module json is a Starlark module of JSON-related functions.") +public final class Json implements StarlarkValue { + + private Json() {} + + /** + * The module instance. You may wish to add this to your predeclared environment under the name + * "json". + */ + public static final Json INSTANCE = new Json(); + + /** An interface for StarlarkValue subclasses to define their own JSON encoding. */ + public interface Encodable { + String encodeJSON(); + } + + /** + * Encodes a Starlark value as JSON. + * + *

An application-defined subclass of StarlarkValue may define its own JSON encoding by + * implementing the {@link Encodable} interface. Otherwise, the encoder tests for the {@link Map}, + * {@link StarlarkIterable}, and {@link Structure} interfaces, in that order, resulting in + * dict-like, list-like, and struct-like encoding, respectively. See the Starlark documentation + * annotation for more detail. + * + *

Encoding any other value yields an error. + */ + @StarlarkMethod( + name = "encode", + doc = + "

The encode function accepts one required positional argument, which it converts to" + + " JSON by cases:\n" + + "

    \n" + + "
  • None, True, and False are converted to 'null', 'true', and 'false'," + + " respectively.\n" + + "
  • An int, no matter how large, is encoded as a decimal integer. Some decoders" + + " may not be able to decode very large integers.\n" + + "
  • A float is encoded using a decimal point or an exponent or both, even if its" + + " numeric value is an integer. It is an error to encode a non-finite " + + " floating-point value.\n" + + "
  • A string value is encoded as a JSON string literal that denotes the value. " + + " Each unpaired surrogate is replaced by U+FFFD.\n" + + "
  • A dict is encoded as a JSON object, in key order. It is an error if any key" + + " is not a string.\n" + + "
  • A list or tuple is encoded as a JSON array.\n" + + "
  • A struct-like value is encoded as a JSON object, in field name order.\n" + + "
\n" + + "An application-defined type may define its own JSON encoding.\n" + + "Encoding any other value yields an error.\n", + parameters = {@Param(name = "x")}) + public String encode(Object x) throws EvalException { + Encoder enc = new Encoder(); + try { + enc.encode(x); + } catch (StackOverflowError unused) { + throw Starlark.errorf("nesting depth limit exceeded"); + } + return enc.out.toString(); + } + + private static final class Encoder { + + private final StringBuilder out = new StringBuilder(); + + private void encode(Object x) throws EvalException { + if (x == Starlark.NONE) { + out.append("null"); + return; + } + + if (x instanceof String s) { + appendQuoted(s); + return; + } + + if (x instanceof Boolean || x instanceof StarlarkInt) { + out.append(x); + return; + } + + if (x instanceof StarlarkFloat fl) { + if (!Double.isFinite(fl.toDouble())) { + throw Starlark.errorf("cannot encode non-finite float %s", x); + } + out.append(x.toString()); // always contains a decimal point or exponent + return; + } + + if (x instanceof Encodable) { + // Application-defined Starlark value types + // may define their own JSON encoding. + out.append(((Encodable) x).encodeJSON()); + return; + } + + // e.g. dict (must have string keys) + if (x instanceof Map m) { + // Sort keys for determinism. + Object[] keys = m.keySet().toArray(); + for (Object key : keys) { + if (!(key instanceof String)) { + throw Starlark.errorf( + "%s has %s key, want string", Starlark.type(x), Starlark.type(key)); + } + } + Arrays.sort(keys); + + // emit object + out.append('{'); + String sep = ""; + for (Object key : keys) { + out.append(sep); + sep = ","; + appendQuoted((String) key); + out.append(':'); + try { + encode(m.get(key)); + } catch (EvalException ex) { + throw Starlark.errorf( + "in %s key %s: %s", Starlark.type(x), Starlark.repr(key), ex.getMessage()); + } + } + out.append('}'); + return; + } + + // e.g. tuple, list + if (x instanceof StarlarkIterable it) { + out.append('['); + String sep = ""; + int i = 0; + for (Object elem : it) { + out.append(sep); + sep = ","; + try { + encode(elem); + } catch (EvalException ex) { + throw Starlark.errorf("at %s index %d: %s", Starlark.type(x), i, ex.getMessage()); + } + i++; + } + out.append(']'); + return; + } + + // e.g. struct + if (x instanceof Structure obj) { + // Sort keys for determinism. + String[] fields = obj.getFieldNames().toArray(new String[0]); + Arrays.sort(fields); + + out.append('{'); + String sep = ""; + for (String field : fields) { + out.append(sep); + sep = ","; + appendQuoted(field); + out.append(":"); + try { + Object v = obj.getValue(field); // may fail (field not defined) + encode(v); // may fail (unexpected type) + } catch (EvalException ex) { + throw Starlark.errorf("in %s field .%s: %s", Starlark.type(x), field, ex.getMessage()); + } + } + out.append('}'); + return; + } + + throw Starlark.errorf("cannot encode %s as JSON", Starlark.type(x)); + } + + private void appendQuoted(String s) { + // We use String's code point iterator so that we can map + // unpaired surrogates to U+FFFD in the output. + // TODO(adonovan): if we ever get an isPrintable(codepoint) + // function, use uXXXX escapes for non-printables. + out.append('"'); + for (int i = 0, n = s.length(); i < n; ) { + int cp = s.codePointAt(i); + + // ASCII control code? + if (cp < 0x20) { + switch (cp) { + case '\b': + out.append("\\b"); + break; + case '\f': + out.append("\\f"); + break; + case '\n': + out.append("\\n"); + break; + case '\r': + out.append("\\r"); + break; + case '\t': + out.append("\\t"); + break; + default: + out.append("\\u00"); + out.append(HEX[(cp >> 4) & 0xf]); + out.append(HEX[cp & 0xf]); + } + i++; + continue; + } + + // printable ASCII (or DEL 0x7f)? (common case) + if (cp < 0x80) { + if (cp == '"' || cp == '\\') { + out.append('\\'); + } + out.append((char) cp); + i++; + continue; + } + + // non-ASCII + if (Character.MIN_SURROGATE <= cp && cp <= Character.MAX_SURROGATE) { + cp = 0xFFFD; // unpaired surrogate + } + out.appendCodePoint(cp); + i += Character.charCount(cp); + } + out.append('"'); + } + } + + private static final char[] HEX = "0123456789abcdef".toCharArray(); + + /** Parses a JSON string as a Starlark value. */ + @StarlarkMethod( + name = "decode", + doc = + "The decode function has one required positional parameter: a JSON string.\n" + + "It returns the Starlark value that the string denotes.\n" + + "
  • \"null\", \"true\" and \"false\"" + + " are parsed as None, True, and False.\n" + + "
  • Numbers are parsed as int, or as a float if they contain a decimal point or an" + + " exponent. Although JSON has no syntax for non-finite values, very large values" + + " may be decoded as infinity.\n" + + "
  • a JSON object is parsed as a new unfrozen Starlark dict. If the same key" + + " string occurs more than once in the object, the last value for the key is kept.\n" + + "
  • a JSON array is parsed as new unfrozen Starlark list.\n" + + "
\n" + + "If x is not a valid JSON encoding and the optional" + + " default parameter is specified (including specified as" + + " None), this function returns the default value.\n" + + "If x is not a valid JSON encoding and the optional" + + " default parameter is not specified, this function fails.", + parameters = { + @Param(name = "x", doc = "JSON string to decode."), + @Param( + name = "default", + named = true, + doc = "If specified, the value to return when x cannot be decoded.", + defaultValue = "unbound") + }, + useStarlarkThread = true) + public Object decode(String x, Object defaultValue, StarlarkThread thread) throws EvalException { + try { + return new Decoder(thread.mutability(), x).decode(); + } catch (EvalException e) { + if (defaultValue != Starlark.UNBOUND) { + return defaultValue; + } else { + throw e; + } + } + } + + private static final class Decoder { + + // The decoder necessarily makes certain representation choices + // such as list vs tuple, struct vs dict, int vs float. + // In principle, we could parameterize it to allow the caller to + // control the returned types, but there's no compelling need yet. + + private final Mutability mu; + private final String s; // the input string + private int i = 0; // current index in s + + private Decoder(Mutability mu, String s) { + this.mu = mu; + this.s = s; + } + + // decode is the entry point into the decoder. + private Object decode() throws EvalException { + try { + Object x = parse(); + if (skipSpace()) { + throw Starlark.errorf("unexpected character %s after value", quoteChar(s.charAt(i))); + } + return x; + } catch (StackOverflowError unused) { + throw Starlark.errorf("nesting depth limit exceeded"); + } catch (EvalException ex) { + throw Starlark.errorf("at offset %d, %s", i, ex.getMessage()); + } + } + + // parse returns the next JSON value from the input. + // It consumes leading but not trailing whitespace. + private Object parse() throws EvalException { + char c = next(); + switch (c) { + case '"': + return parseString(); + + case 'n': + if (s.startsWith("null", i)) { + i += "null".length(); + return Starlark.NONE; + } + break; + + case 't': + if (s.startsWith("true", i)) { + i += "true".length(); + return true; + } + break; + + case 'f': + if (s.startsWith("false", i)) { + i += "false".length(); + return false; + } + break; + + case '[': + // array + StarlarkList list = StarlarkList.newList(mu); + + i++; // '[' + c = next(); + if (c != ']') { + while (true) { + Object elem = parse(); + list.addElement(elem); // can't fail + c = next(); + if (c != ',') { + if (c != ']') { + throw Starlark.errorf("got %s, want ',' or ']'", quoteChar(c)); + } + break; + } + i++; // ',' + } + } + i++; // ']' + return list; + + case '{': + // object + Dict dict = Dict.of(mu); + + i++; // '{' + c = next(); + if (c != '}') { + while (true) { + Object key = parse(); + if (!(key instanceof String)) { + throw Starlark.errorf("got %s for object key, want string", Starlark.type(key)); + } + c = next(); + if (c != ':') { + throw Starlark.errorf("after object key, got %s, want ':' ", quoteChar(c)); + } + i++; // ':' + Object value = parse(); + dict.putEntry((String) key, value); // can't fail + c = next(); + if (c != ',') { + if (c != '}') { + throw Starlark.errorf("in object, got %s, want ',' or '}'", quoteChar(c)); + } + break; + } + i++; // ',' + } + } + i++; // '}' + return dict; + + default: + // number? + if (isdigit(c) || c == '-') { + return parseNumber(c); + } + break; + } + throw Starlark.errorf("unexpected character %s", quoteChar(c)); + } + + private String parseString() throws EvalException { + i++; // '"' + StringBuilder str = new StringBuilder(); + while (i < s.length()) { + char c = s.charAt(i); + + // end quote? + if (c == '"') { + i++; // skip '"' + return str.toString(); + } + + // literal char? + if (c != '\\') { + // reject unescaped control codes + if (c <= 0x1F) { + throw Starlark.errorf("invalid character '\\x%02x' in string literal", (int) c); + } + i++; // consume + str.append(c); + continue; + } + + // escape: uXXXX or [\/bfnrt"] + i++; // '\\' + if (i == s.length()) { + throw Starlark.errorf("incomplete escape"); + } + c = s.charAt(i); + i++; // consume c + switch (c) { + case '\\': + case '/': + case '"': + str.append(c); + break; + case 'b': + str.append('\b'); + break; + case 'f': + str.append('\f'); + break; + case 'n': + str.append('\n'); + break; + case 'r': + str.append('\r'); + break; + case 't': + str.append('\t'); + break; + case 'u': // \ uXXXX + if (i + 4 >= s.length()) { + throw Starlark.errorf("incomplete \\uXXXX escape"); + } + int hex = 0; + for (int j = 0; j < 4; j++) { + c = s.charAt(i + j); + int nybble = 0; + if (isdigit(c)) { + nybble = c - '0'; + } else if ('a' <= c && c <= 'f') { + nybble = 10 + c - 'a'; + } else if ('A' <= c && c <= 'F') { + nybble = 10 + c - 'A'; + } else { + throw Starlark.errorf("invalid hex char %s in \\uXXXX escape", quoteChar(c)); + } + hex = (hex << 4) | nybble; + } + str.append((char) hex); + i += 4; + break; + default: + throw Starlark.errorf("invalid escape '\\%s'", c); + } + } + throw Starlark.errorf("unclosed string literal"); + } + + private Object parseNumber(char c) throws EvalException { + // For now, allow any sequence of [0-9.eE+-]*. + boolean isfloat = false; // whether digit string contains [.Ee+-] (other than leading minus) + int j = i; + for (j = i + 1; j < s.length(); j++) { + c = s.charAt(j); + if (isdigit(c)) { + // ok + } else if (c == '.' || c == 'e' || c == 'E' || c == '+' || c == '-') { + isfloat = true; + } else { + break; + } + } + + String num = s.substring(i, j); + + int digits = i; // s[digits:j] is the digit string + if (s.charAt(i) == '-') { + digits++; + } + + // Structural checks not performed by parse routines below. + // Unlike most C-like languages, + // JSON disallows a leading zero before a digit. + if (digits == j // "-" + || s.charAt(digits) == '.' // ".5" + || s.charAt(j - 1) == '.' // "0." + || num.contains(".e") // "5.e1" + || (s.charAt(digits) == '0' && j - digits > 1 && isdigit(s.charAt(digits + 1)))) { // "01" + throw Starlark.errorf("invalid number: %s", num); + } + + i = j; + + // parse number literal + try { + if (isfloat) { + double x = Double.parseDouble(num); + return StarlarkFloat.of(x); + } else { + return StarlarkInt.parse(num, 10); + } + } catch (NumberFormatException unused) { + throw Starlark.errorf("invalid number: %s", num); + } + } + + // skipSpace consumes leading spaces, and reports whether there is more input. + private boolean skipSpace() { + for (; i < s.length(); i++) { + char c = s.charAt(i); + if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { + return true; + } + } + return false; + } + + // next consumes leading spaces and returns the first non-space. + private char next() throws EvalException { + if (skipSpace()) { + return s.charAt(i); + } + throw Starlark.errorf("unexpected end of file"); + } + } + + @StarlarkMethod( + name = "indent", + doc = + "The indent function returns the indented form of a valid JSON-encoded string.\n" + + "Each array element or object field appears on a new line, beginning with" + + " the prefix string followed by one or more copies of the indent string, according" + + " to its nesting depth.\n" + + "The function accepts one required positional parameter, the JSON string,\n" + + "and two optional keyword-only string parameters, prefix and indent,\n" + + "that specify a prefix of each new line, and the unit of indentation.\n" + + "If the input is not valid, the function may fail or return invalid output.\n", + parameters = { + @Param(name = "s"), + @Param(name = "prefix", positional = false, named = true, defaultValue = "''"), + @Param(name = "indent", positional = false, named = true, defaultValue = "'\\t'") + }) + public String indent(String s, String prefix, String indent) throws EvalException { + // Indentation can be efficiently implemented in a single pass, independent of encoding, + // with no state other than a depth counter. This separation enables efficient indentation + // of values obtained from, say, reading a file, without the need for decoding. + + Indenter in = new Indenter(prefix, indent, s); + try { + in.indent(); + } catch (StringIndexOutOfBoundsException unused) { + throw Starlark.errorf("input is not valid JSON"); + } + return in.out.toString(); + } + + @StarlarkMethod( + name = "encode_indent", + doc = + "The encode_indent function is equivalent to json.indent(json.encode(x)," + + " ...). See indent for description of formatting parameters.", + parameters = { + @Param(name = "x"), + @Param(name = "prefix", positional = false, named = true, defaultValue = "''"), + @Param(name = "indent", positional = false, named = true, defaultValue = "'\\t'"), + }) + public String encodeIndent(Object x, String prefix, String indent) throws EvalException { + return indent(encode(x), prefix, indent); + } + + private static final class Indenter { + + private final StringBuilder out = new StringBuilder(); + private final String prefix; + private final String indent; + private final String s; // input string + private int i; // current index in s, possibly out of bounds + + Indenter(String prefix, String indent, String s) { + this.prefix = prefix; + this.indent = indent; + this.s = s; + } + + // Appends a single JSON value to str. + // May throw StringIndexOutOfBoundsException. + // + // The current implementation is a rudimentary placeholder: + // given invalid JSON, it produces garbage output. + // TODO(adonovan): factor Decoder and Indenter using a + // validating state machine, without loss of efficiency. + // This requires different states after [, {, :, etc, + // and a stack of open tokens. + private void indent() throws EvalException { + int depth = 0; + + // token loop + do { // while (depth > 0) + char c = next(); + int start = i; + switch (c) { + case '"': // string + for (c = s.charAt(++i); c != '"'; c = s.charAt(++i)) { + if (c == '\\') { + c = s.charAt(++i); + if (c == 'u') { + i += 4; + } + } + } + i++; // '"' + out.append(s, start, i); + break; + + case 'n': + i += "null".length(); + out.append(s, start, i); + break; + + case 't': + i += "true".length(); + out.append(s, start, i); + break; + + case 'f': + i += "false".length(); + out.append(s, start, i); + break; + + case ',': + i++; + out.append(','); + newline(depth); + break; + + case '[': + case '{': + i++; + out.append(c); + c = next(); + if (c == ']' || c == '}') { + i++; + out.append(c); + } else { + newline(++depth); + } + break; + + case ']': + case '}': + i++; + newline(--depth); + out.append(c); + break; + + case ':': + i++; + out.append(": "); + break; + + default: + // number + if (!(isdigit(c) || c == '-')) { + throw Starlark.errorf("unexpected character %s", quoteChar(c)); + } + while (i < s.length()) { + c = s.charAt(++i); + if (!(isdigit(c) || c == '.' || c == 'e' || c == 'E' || c == '+' || c == '-')) { + break; + } + } + out.append(s, start, i); + break; + } + } while (depth > 0); + } + + private void newline(int depth) { + out.append('\n').append(prefix); + for (int i = 0; i < depth; i++) { + out.append(indent); + } + } + + // skipSpace consumes leading spaces, and reports whether there is more input. + private boolean skipSpace() { + for (; i < s.length(); i++) { + char c = s.charAt(i); + if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { + return true; + } + } + return false; + } + + // next consumes leading spaces and returns the first non-space. + private char next() throws EvalException { + if (skipSpace()) { + return s.charAt(i); + } + throw Starlark.errorf("unexpected end of file"); + } + } + + private static boolean isdigit(char c) { + return c >= '0' && c <= '9'; + } + + // Returns a Starlark string literal that denotes c. + private static String quoteChar(char c) { + return Starlark.repr("" + c); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/spelling/BUILD b/third_party/bazel/main/java/net/starlark/java/spelling/BUILD new file mode 100644 index 000000000..48164c993 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/spelling/BUILD @@ -0,0 +1,27 @@ +# The spelling checker used by the Starlark interpreter. + +load("@rules_java//java:defs.bzl", "java_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src:__subpackages__"], +) + +java_library( + name = "spelling", + srcs = ["SpellChecker.java"], + # The SpellChecker is available to Starlark and Bazel, but not the world. + visibility = [ + "//src/main/java/net/starlark/java:bazel", + "//src/main/java/net/starlark/java:starlark", + ], + deps = [ + "//third_party:jsr305", + ], +) diff --git a/third_party/bazel/main/java/net/starlark/java/spelling/SpellChecker.java b/third_party/bazel/main/java/net/starlark/java/spelling/SpellChecker.java new file mode 100644 index 000000000..9805b9eeb --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/spelling/SpellChecker.java @@ -0,0 +1,110 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.spelling; + +import javax.annotation.Nullable; + +/** + * Class that provides functions to do spell checking, i.e. detect typos + * and make suggestions. + */ +public final class SpellChecker { + /** + * Computes the edit distance between two strings. The edit distance is + * the minimum number of insertions, deletions and replacements to + * transform a string into the other string. + * + * maxEditDistance is the maximum distance the function can return. If + * it would be greater, the function returns -1. It is useful for + * speeding up the computations. + */ + public static int editDistance(String s1, String s2, int maxEditDistance) { + // This is the Levenshtein distance, as described here: + // http://en.wikipedia.org/wiki/Levenshtein_distance + // + // We don't need to keep the full matrix. To update a cell, we only + // need top-left, top, and left values. Using a single array is + // sufficient. Top value is still in row[j] from the last iteration. + // Top-left value is stored in 'previous'. Left value is row[j - 1]. + + if (s1.equals(s2)) { + return 0; + } + // Short-circuit based on string length. + if (Math.abs(s1.length() - s2.length()) > maxEditDistance) { + return -1; + } + + int[] row = new int[s2.length() + 1]; + for (int i = 0; i <= s2.length(); i++) { + row[i] = i; + } + + for (int i = 1; i <= s1.length(); i++) { + row[0] = i; + int bestInTheRow = row[0]; + int previous = i - 1; + + for (int j = 1; j <= s2.length(); j++) { + int old = row[j]; + + row[j] = Math.min( + previous + (s1.charAt(i - 1) == s2.charAt(j - 1) ? 0 : 1), + 1 + Math.min(row[j - 1], row[j])); + previous = old; + bestInTheRow = Math.min(bestInTheRow, row[j]); + } + if (bestInTheRow > maxEditDistance) { + return -1; + } + } + int result = row[s2.length()]; + return result <= maxEditDistance ? result : -1; + } + + /** + * Find in words which string is the most similar to input (according to + * the edit distance, ignoring case) - or null if no string is similar + * enough. In case of equality, the first one in words wins. + */ + @Nullable + public static String suggest(String input, Iterable words) { + String best = null; + // Heuristic: the expected number of typos depends on the length of the word. + int bestDistance = Math.min(5, (input.length() + 1) / 2); + input = input.toLowerCase(); + for (String candidate : words) { + int d = editDistance(input, candidate.toLowerCase(), bestDistance); + if (d >= 0 && d < bestDistance) { + bestDistance = d; + best = candidate; + } + } + return best; + } + + /** + * Return a string to be used at the end of an error message. It is either an empty string, or a + * spelling suggestion, e.g. " (did you mean 'x'?)". + */ + public static String didYouMean(String input, Iterable words) { + String suggestion = suggest(input, words); + if (suggestion == null) { + return ""; + } else { + return " (did you mean '" + suggestion + "'?)"; + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Argument.java b/third_party/bazel/main/java/net/starlark/java/syntax/Argument.java new file mode 100644 index 000000000..069e3f210 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/Argument.java @@ -0,0 +1,123 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import com.google.common.base.Preconditions; +import javax.annotation.Nullable; + +/** + * Syntax node for an argument to a function. + * + *

Arguments may be of four forms, as in {@code f(expr, id=expr, *expr, **expr)}. These are + * represented by the subclasses Positional, Keyword, Star, and StarStar. + */ +public abstract class Argument extends Node { + + protected final Expression value; + + Argument(FileLocations locs, Expression value) { + super(locs); + this.value = Preconditions.checkNotNull(value); + } + + public final Expression getValue() { + return value; + } + + @Override + public int getEndOffset() { + return value.getEndOffset(); + } + + /** Return the name of this argument's parameter, or null if it is not a Keyword argument. */ + @Nullable + public String getName() { + return null; + } + + /** Syntax node for a positional argument, {@code f(expr)}. */ + public static final class Positional extends Argument { + Positional(FileLocations locs, Expression value) { + super(locs, value); + } + + @Override + public int getStartOffset() { + return value.getStartOffset(); + } + } + + /** Syntax node for a keyword argument, {@code f(id=expr)}. */ + public static final class Keyword extends Argument { + + // Unlike in Python, keyword arguments in Bazel BUILD files + // are about 10x more numerous than positional arguments. + + final Identifier id; + + Keyword(FileLocations locs, Identifier id, Expression value) { + super(locs, value); + this.id = id; + } + + public Identifier getIdentifier() { + return id; + } + + @Override + public String getName() { + return id.getName(); + } + + @Override + public int getStartOffset() { + return id.getStartOffset(); + } + } + + /** Syntax node for an argument of the form {@code f(*expr)}. */ + public static final class Star extends Argument { + private final int starOffset; + + Star(FileLocations locs, int starOffset, Expression value) { + super(locs, value); + this.starOffset = starOffset; + } + + @Override + public int getStartOffset() { + return starOffset; + } + } + + /** Syntax node for an argument of the form {@code f(**expr)}. */ + public static final class StarStar extends Argument { + private final int starStarOffset; + + StarStar(FileLocations locs, int starStarOffset, Expression value) { + super(locs, value); + this.starStarOffset = starStarOffset; + } + + @Override + public int getStartOffset() { + return starStarOffset; + } + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/AssignmentStatement.java b/third_party/bazel/main/java/net/starlark/java/syntax/AssignmentStatement.java new file mode 100644 index 000000000..0145e6775 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/AssignmentStatement.java @@ -0,0 +1,85 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.syntax; + +import javax.annotation.Nullable; + +/** + * Syntax node for an assignment statement ({@code lhs = rhs}) or augmented assignment statement + * ({@code lhs op= rhs}). + */ +public final class AssignmentStatement extends Statement { + + private final Expression lhs; // = IDENTIFIER | DOT | INDEX | LIST_EXPR + @Nullable private final TokenKind op; // TODO(adonovan): make this mandatory even when '='. + private final int opOffset; + private final Expression rhs; + + /** + * Constructs an assignment statement. For an ordinary assignment ({@code op == null}), the LHS + * expression must be of the form {@code id}, {@code x.y}, {@code x[i]}, {@code [e, ...]}, or + * {@code (e, ...)}, where x, i, and e are arbitrary expressions. For an augmented assignment, the + * list and tuple forms are disallowed. + */ + AssignmentStatement( + FileLocations locs, Expression lhs, @Nullable TokenKind op, int opOffset, Expression rhs) { + super(locs, Kind.ASSIGNMENT); + this.lhs = lhs; + this.op = op; + this.opOffset = opOffset; + this.rhs = rhs; + } + + /** Returns the LHS of the assignment. */ + public Expression getLHS() { + return lhs; + } + + /** Returns the operator of an augmented assignment, or null for an ordinary assignment. */ + @Nullable + public TokenKind getOperator() { + return op; + } + + /** Returns the location of the assignment operator. */ + public Location getOperatorLocation() { + return locs.getLocation(opOffset); + } + + @Override + public int getStartOffset() { + return lhs.getStartOffset(); + } + + @Override + public int getEndOffset() { + return rhs.getEndOffset(); + } + + /** Reports whether this is an augmented assignment ({@code getOperator() != null}). */ + public boolean isAugmented() { + return op != null; + } + + /** Returns the RHS of the assignment. */ + public Expression getRHS() { + return rhs; + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/BUILD b/third_party/bazel/main/java/net/starlark/java/syntax/BUILD new file mode 100644 index 000000000..9b13f7867 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/BUILD @@ -0,0 +1,71 @@ +# Bazel's Starlark interpreter + +load("@rules_java//java:defs.bzl", "java_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src:__subpackages__"], +) + +# The Starlark frontend (syntax, scanner, parser, resolver) +java_library( + name = "syntax", + srcs = [ + "Argument.java", + "AssignmentStatement.java", + "BinaryOperatorExpression.java", + "CallExpression.java", + "Comment.java", + "Comprehension.java", + "ConditionalExpression.java", + "DefStatement.java", + "DictExpression.java", + "DotExpression.java", + "Expression.java", + "ExpressionStatement.java", + "FileLocations.java", + "FileOptions.java", + "FloatLiteral.java", + "FlowStatement.java", + "ForStatement.java", + "Identifier.java", + "IfStatement.java", + "IndexExpression.java", + "IntLiteral.java", + "LambdaExpression.java", + "Lexer.java", + "ListExpression.java", + "LoadStatement.java", + "Location.java", + "Node.java", + "NodePrinter.java", + "NodeVisitor.java", + "Parameter.java", + "Parser.java", + "ParserInput.java", + "Program.java", + "Resolver.java", + "ReturnStatement.java", + "SliceExpression.java", + "StarlarkFile.java", + "Statement.java", + "StringLiteral.java", + "SyntaxError.java", + "TokenKind.java", + "UnaryOperatorExpression.java", + ], + visibility = ["//src/main/java/net/starlark/java:clients"], + # Do not add Bazel or Google dependencies here! + deps = [ + "//src/main/java/net/starlark/java/spelling", + "//third_party:auto_value", + "//third_party:guava", + "//third_party:jsr305", + ], +) diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/BinaryOperatorExpression.java b/third_party/bazel/main/java/net/starlark/java/syntax/BinaryOperatorExpression.java new file mode 100644 index 000000000..27699bf30 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/BinaryOperatorExpression.java @@ -0,0 +1,96 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import java.util.EnumSet; + +/** A BinaryExpression represents a binary operator expression 'x op y'. */ +public final class BinaryOperatorExpression extends Expression { + + private final Expression x; + private final TokenKind op; // one of 'operators' + private final int opOffset; + private final Expression y; + + /** operators is the set of valid binary operators. */ + public static final EnumSet operators = + EnumSet.of( + TokenKind.AND, + TokenKind.EQUALS_EQUALS, + TokenKind.GREATER, + TokenKind.GREATER_EQUALS, + TokenKind.IN, + TokenKind.LESS, + TokenKind.LESS_EQUALS, + TokenKind.MINUS, + TokenKind.NOT_EQUALS, + TokenKind.NOT_IN, + TokenKind.OR, + TokenKind.PERCENT, + TokenKind.SLASH, + TokenKind.SLASH_SLASH, + TokenKind.PLUS, + TokenKind.PIPE, + TokenKind.STAR); + + BinaryOperatorExpression( + FileLocations locs, Expression x, TokenKind op, int opOffset, Expression y) { + super(locs, Kind.BINARY_OPERATOR); + this.x = x; + this.op = op; + this.opOffset = opOffset; + this.y = y; + } + + /** Returns the left operand. */ + public Expression getX() { + return x; + } + + /** Returns the operator. */ + public TokenKind getOperator() { + return op; + } + + public Location getOperatorLocation() { + return locs.getLocation(opOffset); + } + + /** Returns the right operand. */ + public Expression getY() { + return y; + } + + @Override + public int getStartOffset() { + return x.getStartOffset(); + } + + @Override + public int getEndOffset() { + return y.getEndOffset(); + } + + @Override + public String toString() { + // This omits the parentheses for brevity, but is not correct in general due to operator + // precedence rules. + return x + " " + op + " " + y; + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/CallExpression.java b/third_party/bazel/main/java/net/starlark/java/syntax/CallExpression.java new file mode 100644 index 000000000..283053c94 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/CallExpression.java @@ -0,0 +1,99 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +/** Syntax node for a function call expression. */ +public final class CallExpression extends Expression { + + private final Expression function; + private final Location lparenLocation; + private final ImmutableList arguments; + private final int rparenOffset; + + private final int numPositionalArgs; + + CallExpression( + FileLocations locs, + Expression function, + Location lparenLocation, + ImmutableList arguments, + int rparenOffset) { + super(locs, Kind.CALL); + this.function = Preconditions.checkNotNull(function); + this.lparenLocation = lparenLocation; + this.arguments = arguments; + this.rparenOffset = rparenOffset; + + int n = 0; + for (Argument arg : arguments) { + if (arg instanceof Argument.Positional) { + n++; + } + } + this.numPositionalArgs = n; + } + + /** Returns the function that is called. */ + public Expression getFunction() { + return this.function; + } + + /** Returns the number of arguments of type {@code Argument.Positional}. */ + public int getNumPositionalArguments() { + return numPositionalArgs; + } + + /** Returns the function arguments. */ + public ImmutableList getArguments() { + return arguments; + } + + @Override + public int getStartOffset() { + return function.getStartOffset(); + } + + @Override + public int getEndOffset() { + return rparenOffset + 1; + } + + public Location getLparenLocation() { + // Unlike all other getXXXLocation methods, this one returns a reference to + // a previously materialized Location. getLparenLocation is unique among + // locations because the tree-walking evaluator needs it frequently even + // in the absence of errors. When we switch to a compiled representation + // we can dispense with this optimization. + return lparenLocation; + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(); + buf.append(function); + buf.append('('); + ListExpression.appendNodes(buf, arguments); + buf.append(')'); + return buf.toString(); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Comment.java b/third_party/bazel/main/java/net/starlark/java/syntax/Comment.java new file mode 100644 index 000000000..b246860f0 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/Comment.java @@ -0,0 +1,52 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +/** Syntax node for comments. */ +public final class Comment extends Node { + + private final int offset; + private final String text; + + Comment(FileLocations locs, int offset, String text) { + super(locs); + this.offset = offset; + this.text = text; + } + + /** Returns the text of the comment, including the leading '#' but not the trailing newline. */ + public String getText() { + return text; + } + + @Override + public int getStartOffset() { + return offset; + } + + @Override + public int getEndOffset() { + return offset + text.length(); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } + + @Override + public String toString() { + return text; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Comprehension.java b/third_party/bazel/main/java/net/starlark/java/syntax/Comprehension.java new file mode 100644 index 000000000..28e2b6c9b --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/Comprehension.java @@ -0,0 +1,165 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.syntax; + +import com.google.common.collect.ImmutableList; + +/** + * Syntax node for list and dict comprehensions. + * + *

A comprehension contains one or more clauses, e.g. [a+d for a in b if c for d in e] contains + * three clauses: "for a in b", "if c", "for d in e". For and If clauses can happen in any order, + * except that the first one has to be a For. + * + *

The code above can be expanded as: + * + * 

+ *   for a in b:
+ *     if c:
+ *       for d in e:
+ *         result.append(a+d)
+ *
+ * + * result is initialized to [] (list) or {} (dict) and is the return value of the whole expression. + */ +public final class Comprehension extends Expression { + + /** For or If */ + public abstract static class Clause extends Node { + Clause(FileLocations locs) { + super(locs); + } + } + + /** A for clause in a comprehension, e.g. "for a in b" in the example above. */ + public static final class For extends Clause { + private final int forOffset; + private final Expression vars; + private final Expression iterable; + + For(FileLocations locs, int forOffset, Expression vars, Expression iterable) { + super(locs); + this.forOffset = forOffset; + this.vars = vars; + this.iterable = iterable; + } + + public Expression getVars() { + return vars; + } + + public Expression getIterable() { + return iterable; + } + + @Override + public int getStartOffset() { + return forOffset; + } + + @Override + public int getEndOffset() { + return iterable.getEndOffset(); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } + } + + /** A if clause in a comprehension, e.g. "if c" in the example above. */ + public static final class If extends Clause { + private final int ifOffset; + private final Expression condition; + + If(FileLocations locs, int ifOffset, Expression condition) { + super(locs); + this.ifOffset = ifOffset; + this.condition = condition; + } + + public Expression getCondition() { + return condition; + } + + @Override + public int getStartOffset() { + return ifOffset; + } + + @Override + public int getEndOffset() { + return condition.getEndOffset(); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } + } + + private final boolean isDict; // {k: v for vars in iterable} + private final int lbracketOffset; + private final Node body; // Expression or DictExpression.Entry + private final ImmutableList clauses; + private final int rbracketOffset; + + Comprehension( + FileLocations locs, + boolean isDict, + int lbracketOffset, + Node body, + ImmutableList clauses, + int rbracketOffset) { + super(locs, Kind.COMPREHENSION); + this.isDict = isDict; + this.lbracketOffset = lbracketOffset; + this.body = body; + this.clauses = clauses; + this.rbracketOffset = rbracketOffset; + } + + public boolean isDict() { + return isDict; + } + + /** + * Returns the loop body: an expression for a list comprehension, or a DictExpression.Entry for a + * dict comprehension. + */ + public Node getBody() { + return body; + } + + public ImmutableList getClauses() { + return clauses; + } + + @Override + public int getStartOffset() { + return lbracketOffset; + } + + @Override + public int getEndOffset() { + return rbracketOffset + 1; + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/ConditionalExpression.java b/third_party/bazel/main/java/net/starlark/java/syntax/ConditionalExpression.java new file mode 100644 index 000000000..7508df88d --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/ConditionalExpression.java @@ -0,0 +1,57 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +/** Syntax node for an expression of the form {@code t if cond else f}. */ +public final class ConditionalExpression extends Expression { + + private final Expression t; + private final Expression cond; + private final Expression f; + + public Expression getThenCase() { + return t; + } + + public Expression getCondition() { + return cond; + } + + public Expression getElseCase() { + return f; + } + + /** Constructor for a conditional expression */ + ConditionalExpression(FileLocations locs, Expression t, Expression cond, Expression f) { + super(locs, Kind.CONDITIONAL); + this.t = t; + this.cond = cond; + this.f = f; + } + + @Override + public int getStartOffset() { + return t.getStartOffset(); + } + + @Override + public int getEndOffset() { + return f.getEndOffset(); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/DefStatement.java b/third_party/bazel/main/java/net/starlark/java/syntax/DefStatement.java new file mode 100644 index 000000000..884010dc2 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/DefStatement.java @@ -0,0 +1,91 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import javax.annotation.Nullable; + +/** Syntax node for a 'def' statement, which defines a function. */ +public final class DefStatement extends Statement { + + private final int defOffset; + private final Identifier identifier; + private final ImmutableList body; // non-empty if well formed + private final ImmutableList parameters; + + // set by resolver + @Nullable private Resolver.Function resolved; + + DefStatement( + FileLocations locs, + int defOffset, + Identifier identifier, + ImmutableList parameters, + ImmutableList body) { + super(locs, Kind.DEF); + this.defOffset = defOffset; + this.identifier = identifier; + this.parameters = Preconditions.checkNotNull(parameters); + this.body = Preconditions.checkNotNull(body); + } + + @Override + public String toString() { + // "def f(...): \n" + StringBuilder buf = new StringBuilder(); + new NodePrinter(buf).printDefSignature(this); + buf.append(" ...\n"); + return buf.toString(); + } + + public Identifier getIdentifier() { + return identifier; + } + + public ImmutableList getBody() { + return body; + } + + public ImmutableList getParameters() { + return parameters; + } + + void setResolvedFunction(Resolver.Function resolved) { + this.resolved = resolved; + } + + /** Returns information about the resolved function. Set by the resolver. */ + @Nullable + public Resolver.Function getResolvedFunction() { + return resolved; + } + + @Override + public int getStartOffset() { + return defOffset; + } + + @Override + public int getEndOffset() { + return body.isEmpty() + ? identifier.getEndOffset() // wrong, but tree is ill formed + : body.get(body.size() - 1).getEndOffset(); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/DictExpression.java b/third_party/bazel/main/java/net/starlark/java/syntax/DictExpression.java new file mode 100644 index 000000000..24d395dd2 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/DictExpression.java @@ -0,0 +1,93 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import com.google.common.collect.ImmutableList; +import java.util.List; + +/** Syntax node for dict expressions. */ +public final class DictExpression extends Expression { + + /** A key/value pair in a dict expression or comprehension. */ + public static final class Entry extends Node { + + private final Expression key; + private final int colonOffset; + private final Expression value; + + Entry(FileLocations locs, Expression key, int colonOffset, Expression value) { + super(locs); + this.key = key; + this.colonOffset = colonOffset; + this.value = value; + } + + public Expression getKey() { + return key; + } + + public Expression getValue() { + return value; + } + + @Override + public int getStartOffset() { + return key.getStartOffset(); + } + + @Override + public int getEndOffset() { + return value.getEndOffset(); + } + + public Location getColonLocation() { + return locs.getLocation(colonOffset); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } + } + + private final int lbraceOffset; + private final ImmutableList entries; + private final int rbraceOffset; + + DictExpression(FileLocations locs, int lbraceOffset, List entries, int rbraceOffset) { + super(locs, Kind.DICT_EXPR); + this.lbraceOffset = lbraceOffset; + this.entries = ImmutableList.copyOf(entries); + this.rbraceOffset = rbraceOffset; + } + + @Override + public int getStartOffset() { + return lbraceOffset; + } + + @Override + public int getEndOffset() { + return rbraceOffset + 1; + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } + + public ImmutableList getEntries() { + return entries; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/DotExpression.java b/third_party/bazel/main/java/net/starlark/java/syntax/DotExpression.java new file mode 100644 index 000000000..2e93cf17e --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/DotExpression.java @@ -0,0 +1,56 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +/** Syntax node for a dot expression. e.g. obj.field, but not obj.method() */ +public final class DotExpression extends Expression { + + private final Expression object; + private final int dotOffset; + private final Identifier field; + + DotExpression(FileLocations locs, Expression object, int dotOffset, Identifier field) { + super(locs, Kind.DOT); + this.object = object; + this.dotOffset = dotOffset; + this.field = field; + } + + public Expression getObject() { + return object; + } + + public Identifier getField() { + return field; + } + + @Override + public int getStartOffset() { + return object.getStartOffset(); + } + + @Override + public int getEndOffset() { + return field.getEndOffset(); + } + + public Location getDotLocation() { + return locs.getLocation(dotOffset); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Expression.java b/third_party/bazel/main/java/net/starlark/java/syntax/Expression.java new file mode 100644 index 000000000..fa66a18e6 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/Expression.java @@ -0,0 +1,76 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +/** + * Base class for all expression nodes in the AST. + * + *

The only expressions permitted on the left-hand side of an assignment (such as 'lhs=rhs' or + * 'for lhs in expr') are identifiers, dot expressions (x.y), list expressions ([expr, ...]), tuple + * expressions ((expr, ...)), or parenthesized variants of those. In particular and unlike Python, + * slice expressions and starred expressions cannot appear on the LHS. TODO(bazel-team): Add support + * for assigning to slices (e.g. a[2:6] = [3]). + */ +public abstract class Expression extends Node { + + /** + * Kind of the expression. This is similar to using instanceof, except that it's more efficient + * and can be used in a switch/case. + */ + public enum Kind { + BINARY_OPERATOR, + COMPREHENSION, + CONDITIONAL, + DICT_EXPR, + DOT, + CALL, + FLOAT_LITERAL, + IDENTIFIER, + INDEX, + INT_LITERAL, + LAMBDA, + LIST_EXPR, + SLICE, + STRING_LITERAL, + UNARY_OPERATOR, + } + + // Materialize kind as a field so its accessor can be non-virtual. + private final Kind kind; + + Expression(FileLocations locs, Kind kind) { + super(locs); + this.kind = kind; + } + + /** + * Kind of the expression. This is similar to using instanceof, except that it's more efficient + * and can be used in a switch/case. + */ + // Final to avoid cost of virtual call (see #12967). + public final Kind kind() { + return kind; + } + + /** Parses an expression. */ + public static Expression parse(ParserInput input, FileOptions options) + throws SyntaxError.Exception { + return Parser.parseExpression(input, options); + } + + /** Parses an expression with default options. */ + public static Expression parse(ParserInput input) throws SyntaxError.Exception { + return parse(input, FileOptions.DEFAULT); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/ExpressionStatement.java b/third_party/bazel/main/java/net/starlark/java/syntax/ExpressionStatement.java new file mode 100644 index 000000000..e54dcfdde --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/ExpressionStatement.java @@ -0,0 +1,45 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.syntax; + +/** Syntax node for a statement consisting of an expression evaluated for effect. */ +public final class ExpressionStatement extends Statement { + + private final Expression expression; + + ExpressionStatement(FileLocations locs, Expression expression) { + super(locs, Kind.EXPRESSION); + this.expression = expression; + } + + public Expression getExpression() { + return expression; + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } + + @Override + public int getStartOffset() { + return expression.getStartOffset(); + } + + @Override + public int getEndOffset() { + return expression.getEndOffset(); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/FileLocations.java b/third_party/bazel/main/java/net/starlark/java/syntax/FileLocations.java new file mode 100644 index 000000000..07556f9bc --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/FileLocations.java @@ -0,0 +1,122 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.syntax; + +import java.util.Arrays; +import java.util.Objects; +import javax.annotation.concurrent.Immutable; + +/** + * FileLocations maps each source offset within a file to a Location. An offset is a (UTF-16) char + * index such that {@code 0 <= offset <= size}. A Location is a (file, line, column) triple. + */ +@Immutable +final class FileLocations { + + private final int[] linestart; // maps line number (line >= 1) to char offset + private final String file; + private final int size; // size of file in chars + + private FileLocations(int[] linestart, String file, int size) { + this.linestart = linestart; + this.file = file; + this.size = size; + } + + static FileLocations create(char[] buffer, String file) { + return new FileLocations(computeLinestart(buffer), file, buffer.length); + } + + String file() { + return file; + } + + private int getLineAt(int offset) { + if (offset < 0 || offset > size) { + throw new IllegalStateException("Illegal position: " + offset); + } + int lowBoundary = 1; + int highBoundary = linestart.length - 1; + while (true) { + if ((highBoundary - lowBoundary) <= 1) { + if (linestart[highBoundary] > offset) { + return lowBoundary; + } else { + return highBoundary; + } + } + int medium = lowBoundary + ((highBoundary - lowBoundary) >> 1); + if (linestart[medium] > offset) { + highBoundary = medium; + } else { + lowBoundary = medium; + } + } + } + + Location getLocation(int offset) { + int line = getLineAt(offset); + int column = offset - linestart[line] + 1; + return new Location(file, line, column); + } + + int size() { + return size; + } + + @Override + public int hashCode() { + return Objects.hash(Arrays.hashCode(linestart), file, size); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof FileLocations)) { + return false; + } + FileLocations that = (FileLocations) other; + return this.size == that.size + && Arrays.equals(this.linestart, that.linestart) + && this.file.equals(that.file); + } + + private static int[] computeLinestart(char[] buffer) { + // Compute the size. + int size = 2; + for (int i = 0; i < buffer.length; i++) { + if (buffer[i] == '\n') { + size++; + } + } + int[] linestart = new int[size]; + + int index = 0; + linestart[index++] = 0; // The 0th line does not exist - so we fill something in + // to make sure the start pos for the 1st line ends up at + // linestart[1]. Using 0 is useful for tables that are + // completely empty. + linestart[index++] = 0; // The first line ("line 1") starts at offset 0. + + // Scan the buffer and record the offset of each line start. Doing this + // once upfront is faster than checking each char as it is pulled from + // the buffer. + for (int i = 0; i < buffer.length; i++) { + if (buffer[i] == '\n') { + linestart[index++] = i + 1; + } + } + return linestart; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/FileOptions.java b/third_party/bazel/main/java/net/starlark/java/syntax/FileOptions.java new file mode 100644 index 000000000..a7111dc78 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/FileOptions.java @@ -0,0 +1,109 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.syntax; + +import com.google.auto.value.AutoValue; + +/** + * FileOptions is a set of options that affect the static processing---scanning, parsing, validation + * (identifier resolution), and compilation---of a single Starlark file. These options affect the + * language accepted by the frontend (in effect, the dialect), and "code generation", analogous to + * the command-line options of a typical compiler. + * + *

Different files within the same application and even executed within the same thread may be + * subject to different file options. For example, in Bazel, load statements in WORKSPACE files may + * need to be interleaved with other statements, whereas in .bzl files, load statements must appear + * all at the top. A single thread may execute a WORKSPACE file and call functions defined in .bzl + * files. + * + *

The {@link #DEFAULT} options represent the desired behavior for new uses of Starlark. It is a + * goal to keep this set of options small and closed. Each represents a language feature, perhaps a + * deprecated, obscure, or regrettable one. By contrast, {@link StarlarkSemantics} defines a + * (soon-to-be) open-ended set of options that affect the dynamic behavior of Starlark threads and + * (mostly application-defined) built-in functions, and particularly attribute selection operations + * {@code x.f}. + */ +@AutoValue +public abstract class FileOptions { + + /** The default options for Starlark static processing. New clients should use these defaults. */ + public static final FileOptions DEFAULT = builder().build(); + + /** + * During resolution, permit load statements to access private names such as {@code _x}.
+ * (Required for continued support of Bazel "WORKSPACE.resolved" files.) + */ + public abstract boolean allowLoadPrivateSymbols(); + + /** + * During resolution, permit multiple assignments to a given top-level binding, whether file-local + * or global. However, as usual, you may not create both a file-local and a global binding of the + * same name (e.g. {@code load(..., x="x"); x=1}), so if you use this option, you probably want + * {@link #loadBindsGlobally} too, to avoid confusing errors.
+ * (Required for continued support of Bazel BUILD files and Copybara files.) + */ + public abstract boolean allowToplevelRebinding(); + + /** + * During resolution, make load statements bind global variables of the module, not file-local + * variables.
+ * (Intended for use in REPLs, and the Bazel prelude; and in Bazel BUILD files, which make + * frequent use of {@code load(..., "x"); x=1} for reasons unclear.) + */ + public abstract boolean loadBindsGlobally(); + + /** + * During resolution, require load statements to appear before other kinds of statements.
+ * (Required for continued support of Bazel BUILD and especially WORKSPACE files.) + */ + public abstract boolean requireLoadStatementsFirst(); + + /** + * During lexing, whether to ban non-ASCII characters (i.e., characters with code point > U+7F) in + * string literals. + * + *

This applies to string literals' raw content as well as escape sequences. + */ + public abstract boolean stringLiteralsAreAsciiOnly(); + + public static Builder builder() { + // These are the DEFAULT values. + return new AutoValue_FileOptions.Builder() + .allowLoadPrivateSymbols(false) + .allowToplevelRebinding(false) + .loadBindsGlobally(false) + .requireLoadStatementsFirst(true) + .stringLiteralsAreAsciiOnly(false); + } + + public abstract Builder toBuilder(); + + /** This javadoc comment states that FileOptions.Builder is a builder for FileOptions. */ + @AutoValue.Builder + public abstract static class Builder { + // AutoValue why u make me say it 3 times? + public abstract Builder allowLoadPrivateSymbols(boolean value); + + public abstract Builder allowToplevelRebinding(boolean value); + + public abstract Builder loadBindsGlobally(boolean value); + + public abstract Builder requireLoadStatementsFirst(boolean value); + + public abstract Builder stringLiteralsAreAsciiOnly(boolean value); + + public abstract FileOptions build(); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/FloatLiteral.java b/third_party/bazel/main/java/net/starlark/java/syntax/FloatLiteral.java new file mode 100644 index 000000000..13c9d3dde --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/FloatLiteral.java @@ -0,0 +1,53 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +/** Syntax node for a float literal. */ +public final class FloatLiteral extends Expression { + private final String raw; + private final int tokenOffset; + private final double value; + + FloatLiteral(FileLocations locs, String raw, int tokenOffset, double value) { + super(locs, Kind.FLOAT_LITERAL); + this.raw = raw; + this.tokenOffset = tokenOffset; + this.value = value; + } + + /** Returns the value denoted by this literal. */ + public double getValue() { + return value; + } + + /** Returns the raw source text of the literal. */ + public String getRaw() { + return raw; + } + + @Override + public int getStartOffset() { + return tokenOffset; + } + + @Override + public int getEndOffset() { + return tokenOffset + raw.length(); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/FlowStatement.java b/third_party/bazel/main/java/net/starlark/java/syntax/FlowStatement.java new file mode 100644 index 000000000..ae121269f --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/FlowStatement.java @@ -0,0 +1,56 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +/** A class for flow statements (break, continue, and pass) */ +public final class FlowStatement extends Statement { + + private final TokenKind flowKind; // BREAK | CONTINUE | PASS + private final int offset; + + /** + * Constructs a new flow control statement. + * + * @param flowKind The specific kind of flow control statement (break, continue, or pass) + */ + FlowStatement(FileLocations locs, TokenKind flowKind, int offset) { + super(locs, Kind.FLOW); + this.flowKind = flowKind; + this.offset = offset; + } + + public TokenKind getFlowKind() { + return flowKind; + } + + @Override + public String toString() { + return flowKind.toString() + "\n"; + } + + @Override + public int getStartOffset() { + return offset; + } + + @Override + public int getEndOffset() { + return offset + flowKind.toString().length(); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/ForStatement.java b/third_party/bazel/main/java/net/starlark/java/syntax/ForStatement.java new file mode 100644 index 000000000..cd1abda03 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/ForStatement.java @@ -0,0 +1,81 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +/** Syntax node for a for loop statement, {@code for vars in iterable: ...}. */ +public final class ForStatement extends Statement { + + private final int forOffset; + private final Expression vars; + private final Expression iterable; + private final ImmutableList body; // non-empty if well formed + + /** Constructs a for loop statement. */ + ForStatement( + FileLocations locs, + int forOffset, + Expression vars, + Expression iterable, + ImmutableList body) { + super(locs, Kind.FOR); + this.forOffset = forOffset; + this.vars = Preconditions.checkNotNull(vars); + this.iterable = Preconditions.checkNotNull(iterable); + this.body = body; + } + + /** + * Returns variables assigned by each iteration. May be a compound target such as {@code (a[b], + * c.d)}. + */ + public Expression getVars() { + return vars; + } + + /** Returns the iterable value. */ + // TODO(adonovan): rename to getIterable. + public Expression getCollection() { + return iterable; + } + + /** Returns the statements of the loop body. Non-empty if parsing succeeded. */ + public ImmutableList getBody() { + return body; + } + + @Override + public int getStartOffset() { + return forOffset; + } + + @Override + public int getEndOffset() { + return body.isEmpty() + ? iterable.getEndOffset() // wrong, but tree is ill formed + : body.get(body.size() - 1).getEndOffset(); + } + + @Override + public String toString() { + return "for " + vars + " in " + iterable + ": ...\n"; + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Identifier.java b/third_party/bazel/main/java/net/starlark/java/syntax/Identifier.java new file mode 100644 index 000000000..f8567da9e --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/Identifier.java @@ -0,0 +1,132 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import javax.annotation.Nullable; + +/** Syntax node for an identifier. */ +public final class Identifier extends Expression { + + private final String name; + private final int nameOffset; + + // set by Resolver + @Nullable private Resolver.Binding binding; + + Identifier(FileLocations locs, String name, int nameOffset) { + super(locs, Kind.IDENTIFIER); + this.name = name; + this.nameOffset = nameOffset; + } + + @Override + public int getStartOffset() { + return nameOffset; + } + + @Override + public int getEndOffset() { + return nameOffset + name.length(); + } + + /** + * Returns the name of the Identifier. If there were parse errors, misparsed regions may be + * represented as an Identifier for which {@code !isValid(getName())}. + */ + public String getName() { + return name; + } + + public boolean isPrivate() { + return name.startsWith("_"); + } + + /** Returns information about the binding that the identifier denotes. Set by the resolver. */ + @Nullable + public Resolver.Binding getBinding() { + return binding; + } + + void setBinding(Resolver.Binding bind) { + Preconditions.checkState(this.binding == null); + this.binding = bind; + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } + + /** Reports whether the string is a valid identifier. */ + public static boolean isValid(String name) { + // Keep consistent with Lexer.scanIdentifier. + for (int i = 0; i < name.length(); i++) { + char c = name.charAt(i); + if (!(('a' <= c && c <= 'z') + || ('A' <= c && c <= 'Z') + || (i > 0 && '0' <= c && c <= '9') + || (c == '_'))) { + return false; + } + } + return !name.isEmpty(); + } + + /** + * Returns all names bound by an LHS expression. + * + *

Examples: + * + *

    + *
  • <{@code x = ...} binds x. + *
  • <{@code x, [y,z] = ..} binds x, y, z. + *
  • <{@code x[5] = ..} does not bind any names. + *
+ */ + // TODO(adonovan): remove this function in due course. + // - Resolver makes one pass to discover bindings than another to resolve uses. + // When it works in a single pass, it is more efficient to process bindings in order, + // deferring (rare) forward references until the end of the block. + // - Eval calls boundIdentifiers for comprehensions. This can be eliminated when + // variables are assigned frame slot indices. + // - Eval calls boundIdentifiers for the 'export' hack. This can be eliminated + // when we switch to compilation by emitting EXPORT instructions for the necessary + // bindings. (Ideally we would eliminate Bazel's export hack entirely.) + public static ImmutableSet boundIdentifiers(Expression expr) { + if (expr instanceof Identifier) { + // Common case/fast path - skip the builder. + return ImmutableSet.of((Identifier) expr); + } else { + ImmutableSet.Builder result = ImmutableSet.builder(); + collectBoundIdentifiers(expr, result); + return result.build(); + } + } + + private static void collectBoundIdentifiers( + Expression lhs, ImmutableSet.Builder result) { + if (lhs instanceof Identifier) { + result.add((Identifier) lhs); + return; + } + if (lhs instanceof ListExpression variables) { + for (Expression expression : variables.getElements()) { + collectBoundIdentifiers(expression, result); + } + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/IfStatement.java b/third_party/bazel/main/java/net/starlark/java/syntax/IfStatement.java new file mode 100644 index 000000000..7acb65a0f --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/IfStatement.java @@ -0,0 +1,92 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import javax.annotation.Nullable; + +/** Syntax node for an if or elif statement. */ +public final class IfStatement extends Statement { + + private final TokenKind token; // IF or ELIF + private final int ifOffset; + private final Expression condition; + // These blocks may be non-null but empty after a misparse: + private final ImmutableList thenBlock; // non-empty + @Nullable ImmutableList elseBlock; // non-empty if non-null; set after construction + + IfStatement( + FileLocations locs, + TokenKind token, + int ifOffset, + Expression condition, + List thenBlock) { + super(locs, Kind.IF); + this.token = token; + this.ifOffset = ifOffset; + this.condition = condition; + this.thenBlock = ImmutableList.copyOf(thenBlock); + } + + /** + * Reports whether this is an 'elif' statement. + * + *

An elif statement may appear only as the sole statement in the "else" block of another + * IfStatement. + */ + public boolean isElif() { + return token == TokenKind.ELIF; + } + + public Expression getCondition() { + return condition; + } + + public ImmutableList getThenBlock() { + return thenBlock; + } + + @Nullable + public ImmutableList getElseBlock() { + return elseBlock; + } + + void setElseBlock(ImmutableList elseBlock) { + this.elseBlock = elseBlock; + } + + @Override + public int getStartOffset() { + return ifOffset; + } + + @Override + public int getEndOffset() { + List body = elseBlock != null ? elseBlock : thenBlock; + return body.isEmpty() + ? condition.getEndOffset() // wrong, but tree is ill formed + : body.get(body.size() - 1).getEndOffset(); + } + + @Override + public String toString() { + return String.format("if %s: ...\n", condition); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/IndexExpression.java b/third_party/bazel/main/java/net/starlark/java/syntax/IndexExpression.java new file mode 100644 index 000000000..5d41861f6 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/IndexExpression.java @@ -0,0 +1,67 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +/** + * An index expression ({@code obj[field]}). Not to be confused with a slice expression ({@code + * obj[from:to]}). The object may be either a sequence or an associative mapping (most commonly + * lists and dictionaries). + */ +public final class IndexExpression extends Expression { + + private final Expression object; + private final int lbracketOffset; + private final Expression key; + private final int rbracketOffset; + + IndexExpression( + FileLocations locs, + Expression object, + int lbracketOffset, + Expression key, + int rbracketOffset) { + super(locs, Kind.INDEX); + this.object = object; + this.lbracketOffset = lbracketOffset; + this.key = key; + this.rbracketOffset = rbracketOffset; + } + + public Expression getObject() { + return object; + } + + public Expression getKey() { + return key; + } + + @Override + public int getStartOffset() { + return object.getStartOffset(); + } + + @Override + public int getEndOffset() { + return rbracketOffset + 1; + } + + public Location getLbracketLocation() { + return locs.getLocation(lbracketOffset); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/IntLiteral.java b/third_party/bazel/main/java/net/starlark/java/syntax/IntLiteral.java new file mode 100644 index 000000000..0175c53d2 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/IntLiteral.java @@ -0,0 +1,102 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import java.math.BigInteger; + +/** Syntax node for an int literal. */ +public final class IntLiteral extends Expression { + private final String raw; + private final int tokenOffset; + private final Number value; // = Integer | Long | BigInteger + + IntLiteral(FileLocations locs, String raw, int tokenOffset, Number value) { + super(locs, Kind.INT_LITERAL); + this.raw = raw; + this.tokenOffset = tokenOffset; + this.value = value; + } + + /** + * Returns the value denoted by this literal as an Integer, Long, or BigInteger, using the + * narrowest type capable of exactly representing the value. + */ + public Number getValue() { + return value; + } + + /** Returns the raw source text of the literal. */ + public String getRaw() { + return raw; + } + + @Override + public int getStartOffset() { + return tokenOffset; + } + + @Override + public int getEndOffset() { + return tokenOffset + raw.length(); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } + + /** + * Returns the value denoted by a non-negative integer literal with an optional base prefix (but + * no +/- sign), using the narrowest type of Integer, Long, or BigInteger capable of exactly + * representing the value. + * + * @throws NumberFormatException if the string is not a valid literal. + */ + public static Number scan(String str) { + String orig = str; + int radix = 10; + if (str.length() > 1 && str.charAt(0) == '0') { + switch (str.charAt(1)) { + case 'x': + case 'X': + radix = 16; + str = str.substring(2); + break; + case 'o': + case 'O': + radix = 8; + str = str.substring(2); + break; + default: + throw new NumberFormatException( + "invalid octal literal: " + str + " (use '0o" + str.substring(1) + "')"); + } + } + + try { + long v = Long.parseLong(str, radix); + if (v == (int) v) { + return (int) v; + } + return v; + } catch (NumberFormatException unused) { + /* fall through */ + } + try { + return new BigInteger(str, radix); + } catch (NumberFormatException unused) { + throw new NumberFormatException("invalid base-" + radix + " integer literal: " + orig); + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/LambdaExpression.java b/third_party/bazel/main/java/net/starlark/java/syntax/LambdaExpression.java new file mode 100644 index 000000000..19c049a8b --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/LambdaExpression.java @@ -0,0 +1,70 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import javax.annotation.Nullable; + +/** A LambdaExpression ({@code lambda params: body}) denotes an anonymous function. */ +public final class LambdaExpression extends Expression { + + private final int lambdaOffset; // offset of 'lambda' token + private final ImmutableList parameters; + private final Expression body; + + // set by resolver + @Nullable private Resolver.Function resolved; + + LambdaExpression( + FileLocations locs, int lambdaOffset, ImmutableList parameters, Expression body) { + super(locs, Kind.LAMBDA); + this.lambdaOffset = lambdaOffset; + this.parameters = Preconditions.checkNotNull(parameters); + this.body = Preconditions.checkNotNull(body); + } + + public ImmutableList getParameters() { + return parameters; + } + + public Expression getBody() { + return body; + } + + /** Returns information about the resolved function. Set by the resolver. */ + @Nullable + public Resolver.Function getResolvedFunction() { + return resolved; + } + + void setResolvedFunction(Resolver.Function resolved) { + this.resolved = resolved; + } + + @Override + public int getStartOffset() { + return lambdaOffset; + } + + @Override + public int getEndOffset() { + return body.getEndOffset(); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Lexer.java b/third_party/bazel/main/java/net/starlark/java/syntax/Lexer.java new file mode 100644 index 000000000..fcfa1d0eb --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/Lexer.java @@ -0,0 +1,962 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Interner; +import com.google.common.collect.Interners; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Stack; + +/** A scanner for Starlark. */ +final class Lexer { + + // We intern identifiers and keywords to avoid retaining redundant String objects via the AST. + // + // The parser handles interning of string literal values. Benchmarking did not show significant + // benefit to any further internment. See discussion on Google-internal cl/385193833 for details. + private static final Interner identInterner = Interners.newWeakInterner(); + + // --- These fields are accessed directly by the parser: --- + + // Mapping from file offsets to Locations. + final FileLocations locs; + + // Information about current token. Updated by nextToken. + // raw and value are defined only for STRING, INT, FLOAT, IDENTIFIER, and COMMENT. + // TODO(adonovan): rename s/xyz/tokenXyz/ + TokenKind kind; + int start; // start offset + int end; // end offset + Object value; // String, Integer/Long/BigInteger, or Double value of token + + // --- end of parser-visible fields --- + + private final List errors; + + private final FileOptions options; + + // Input buffer and position + private final char[] buffer; + private int pos; + + // The stack of enclosing indentation levels in spaces. + // The first (outermost) element is always zero. + private final Stack indentStack = new Stack<>(); + + private final ImmutableList.Builder comments = ImmutableList.builder(); + + // The number of unclosed open-parens ("(", '{', '[') at the current point in + // the stream. Whitespace is handled differently when this is nonzero. + private int openParenStackDepth = 0; + + // True after a NEWLINE token. In other words, we are outside an + // expression and we have to check the indentation. + private boolean checkIndentation; + + // Number of saved INDENT (>0) or OUTDENT (<0) tokens detected but not yet returned. + private int dents; + + // Characters that can come immediately prior to an '=' character to generate + // a different token + private static final ImmutableMap EQUAL_TOKENS = + ImmutableMap.builder() + .put('=', TokenKind.EQUALS_EQUALS) + .put('!', TokenKind.NOT_EQUALS) + .put('>', TokenKind.GREATER_EQUALS) + .put('<', TokenKind.LESS_EQUALS) + .put('+', TokenKind.PLUS_EQUALS) + .put('-', TokenKind.MINUS_EQUALS) + .put('*', TokenKind.STAR_EQUALS) + .put('/', TokenKind.SLASH_EQUALS) + .put('%', TokenKind.PERCENT_EQUALS) + .put('^', TokenKind.CARET_EQUALS) + .put('&', TokenKind.AMPERSAND_EQUALS) + .put('|', TokenKind.PIPE_EQUALS) + .buildOrThrow(); + + // Constructs a lexer which tokenizes the parser input. + // Errors are appended to errors. + Lexer(ParserInput input, List errors, FileOptions options) { + this.locs = FileLocations.create(input.getContent(), input.getFile()); + this.buffer = input.getContent(); + this.pos = 0; + this.errors = errors; + this.options = options; + this.checkIndentation = true; + this.dents = 0; + + indentStack.push(0); + } + + ImmutableList getComments() { + return comments.build(); + } + + /** + * Reads the next token, updating the Lexer's token fields. It is an error to call nextToken after + * an EOF token. + */ + void nextToken() { + boolean afterNewline = kind == TokenKind.NEWLINE; + tokenize(); + Preconditions.checkState(kind != null); + + // Like Python, always end with a NEWLINE token, even if no '\n' in input: + if (kind == TokenKind.EOF && !afterNewline) { + kind = TokenKind.NEWLINE; + } + } + + private void popParen() { + if (openParenStackDepth == 0) { + // TODO(adonovan): fix: the input ')' should not report an indentation error. + error("indentation error", pos - 1); + } else { + openParenStackDepth--; + } + } + + private void error(String message, int pos) { + errors.add(new SyntaxError(locs.getLocation(pos), message)); + } + + private void setToken(TokenKind kind, int start, int end) { + this.kind = kind; + this.start = start; + this.end = end; + this.value = null; + } + + // setValue sets the value associated with a STRING, FLOAT, INT, + // IDENTIFIER, or COMMENT token, and records the raw text of the token. + private void setValue(Object value) { + this.value = value; + } + + /** Returns the raw input text associated with the current token. */ + String getRaw() { + return bufferSlice(start, end); + } + + /** + * Parses an end-of-line sequence, handling statement indentation correctly. + * + *

UNIX newlines are assumed (LF). Carriage returns are always ignored. + */ + private void newline() { + if (openParenStackDepth > 0) { + newlineInsideExpression(); // in an expression: ignore space + } else { + checkIndentation = true; + setToken(TokenKind.NEWLINE, pos - 1, pos); + } + } + + private void newlineInsideExpression() { + while (pos < buffer.length) { + switch (buffer[pos]) { + case ' ': case '\t': case '\r': + pos++; + break; + default: + return; + } + } + } + + /** Computes indentation (updates dent) and advances pos. */ + private void computeIndentation() { + // we're in a stmt: suck up space at beginning of next line + int indentLen = 0; + while (pos < buffer.length) { + char c = buffer[pos]; + if (c == ' ') { + indentLen++; + pos++; + } else if (c == '\r') { + pos++; + } else if (c == '\t') { + indentLen++; + pos++; + error("Tab characters are not allowed for indentation. Use spaces instead.", pos); + } else if (c == '\n') { // entirely blank line: discard + indentLen = 0; + pos++; + } else if (c == '#') { // line containing only indented comment + int oldPos = pos; + while (pos < buffer.length && c != '\n') { + c = buffer[pos++]; + } + addComment(oldPos, pos - 1); + indentLen = 0; + } else { // printing character + break; + } + } + + if (pos == buffer.length) { + indentLen = 0; + } // trailing space on last line + + int peekedIndent = indentStack.peek(); + if (peekedIndent < indentLen) { // push a level + indentStack.push(indentLen); + dents++; + + } else if (peekedIndent > indentLen) { // pop one or more levels + while (peekedIndent > indentLen) { + indentStack.pop(); + dents--; + peekedIndent = indentStack.peek(); + } + + if (peekedIndent < indentLen) { + error("indentation error", pos - 1); + } + } + } + + /** + * Returns true if current position is in the middle of a triple quote + * delimiter (3 x quot), and advances 'pos' by two if so. + */ + private boolean skipTripleQuote(char quot) { + if (peek(0) == quot && peek(1) == quot) { + pos += 2; + return true; + } else { + return false; + } + } + + /** + * Scans a string literal delimited by 'quot', containing escape sequences. + * + *

ON ENTRY: 'pos' is 1 + the index of the first delimiter + * ON EXIT: 'pos' is 1 + the index of the last delimiter. + */ + private void escapedStringLiteral(char quot, boolean isRaw) { + int literalStartPos = isRaw ? pos - 2 : pos - 1; + boolean inTriplequote = skipTripleQuote(quot); + // more expensive second choice that expands escaped into a buffer + StringBuilder literal = new StringBuilder(); + while (pos < buffer.length) { + char c = buffer[pos]; + pos++; + switch (c) { + case '\n': + if (inTriplequote) { + literal.append(c); + break; + } else { + error("unclosed string literal", literalStartPos); + setToken(TokenKind.STRING, literalStartPos, pos); + setValue(literal.toString()); + return; + } + case '\\': + if (pos == buffer.length) { + error("unclosed string literal", literalStartPos); + setToken(TokenKind.STRING, literalStartPos, pos); + setValue(literal.toString()); + return; + } + if (isRaw) { + // Insert \ and the following character. + // As in Python, it means that a raw string can never end with a single \. + literal.append('\\'); + if (peek(0) == '\r' && peek(1) == '\n') { + literal.append("\n"); + pos += 2; + } else if (buffer[pos] == '\r' || buffer[pos] == '\n') { + literal.append("\n"); + pos += 1; + } else { + literal.append(buffer[pos]); + pos += 1; + } + break; + } + c = buffer[pos]; + pos++; + switch (c) { + case '\r': + if (peek(0) == '\n') { + pos += 1; + break; + } else { + break; + } + case '\n': + // ignore end of line character + break; + case 'a': + literal.append('\u0007'); + break; + case 'b': + literal.append('\b'); + break; + case 'f': + literal.append('\f'); + break; + case 'n': + literal.append('\n'); + break; + case 'r': + literal.append('\r'); + break; + case 't': + literal.append('\t'); + break; + case 'v': + literal.append('\u000b'); + break; + case '\\': + literal.append('\\'); + break; + case '\'': + literal.append('\''); + break; + case '"': + literal.append('"'); + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + { // octal escape + int octal = c - '0'; + if (pos < buffer.length) { + c = buffer[pos]; + if (c >= '0' && c <= '7') { + pos++; + octal = (octal << 3) | (c - '0'); + if (pos < buffer.length) { + c = buffer[pos]; + if (c >= '0' && c <= '7') { + pos++; + octal = (octal << 3) | (c - '0'); + } + } + } + } + if (octal > 0xff) { + error("octal escape sequence out of range (maximum is \\377)", pos - 1); + } else if (options.stringLiteralsAreAsciiOnly() && octal >= 0x80) { + error("octal escape sequence denotes non-ASCII character", pos - 1); + } + literal.append((char) (octal & 0xff)); + break; + } + case 'N': + case 'u': + case 'U': + default: + // unknown char escape => "\literal" + error("invalid escape sequence: \\" + c + ". Use '\\\\' to insert '\\'.", pos - 1); + literal.append('\\'); + literal.append(c); + break; + } + break; + case '\'': + case '"': + if (c != quot || (inTriplequote && !skipTripleQuote(quot))) { + // Non-matching quote, treat it like a regular char. + literal.append(c); + } else { + // Matching close-delimiter, all done. + setToken(TokenKind.STRING, literalStartPos, pos); + setValue(literal.toString()); + return; + } + break; + default: + literal.append(c); + if (options.stringLiteralsAreAsciiOnly() && c >= 0x80) { + error("string literal contains non-ASCII character", pos - 1); + } + break; + } + } + error("unclosed string literal", literalStartPos); + setToken(TokenKind.STRING, literalStartPos, pos); + setValue(literal.toString()); + } + + /** + * Scans a string literal delimited by 'quot'. + * + *

    + *
  • ON ENTRY: 'pos' is 1 + the index of the first delimiter + *
  • ON EXIT: 'pos' is 1 + the index of the last delimiter. + *
+ * + * @param isRaw if true, do not escape the string. + */ + private void stringLiteral(char quot, boolean isRaw) { + int literalStartPos = isRaw ? pos - 2 : pos - 1; + int contentStartPos = pos; + + // Don't even attempt to parse triple-quotes here. + if (skipTripleQuote(quot)) { + pos -= 2; + escapedStringLiteral(quot, isRaw); + return; + } + + // first quick optimistic scan for a simple non-escaped string + while (pos < buffer.length) { + char c = buffer[pos++]; + switch (c) { + case '\n': + error("unclosed string literal", literalStartPos); + setToken(TokenKind.STRING, literalStartPos, pos); + setValue(bufferSlice(contentStartPos, pos - 1)); + return; + case '\\': + if (isRaw) { + if (peek(0) == '\r' && peek(1) == '\n') { + // There was a CRLF after the newline. No shortcut possible, since it needs to be + // transformed into a single LF. + pos = contentStartPos; + escapedStringLiteral(quot, true); + return; + } else { + pos++; + break; + } + } + // oops, hit an escape, need to start over & build a new string buffer + pos = contentStartPos; + escapedStringLiteral(quot, false); + return; + case '\'': + case '"': + if (c == quot) { + // close-quote, all done. + setToken(TokenKind.STRING, literalStartPos, pos); + setValue(bufferSlice(contentStartPos, pos - 1)); + // If we're requiring ASCII-only, do another scan for validation. + if (options.stringLiteralsAreAsciiOnly()) { + for (int i = contentStartPos; i < pos - 1; i++) { + if (buffer[i] >= 0x80) { + // Can report multiple errors per string literal. + error("string literal contains non-ASCII character", i); + } + } + } + return; + } + break; + default: // fall out + } + } + + // If the current position is beyond the end of the file, need to move it backwards + // Possible if the file ends with `r"\` (unclosed raw string literal with a backslash) + if (pos > buffer.length) { + pos = buffer.length; + } + + error("unclosed string literal", literalStartPos); + setToken(TokenKind.STRING, literalStartPos, pos); + setValue(bufferSlice(contentStartPos, pos)); + } + + private static final Map keywordMap = new HashMap<>(); + + static { + keywordMap.put("and", TokenKind.AND); + keywordMap.put("as", TokenKind.AS); + keywordMap.put("assert", TokenKind.ASSERT); + keywordMap.put("break", TokenKind.BREAK); + keywordMap.put("class", TokenKind.CLASS); + keywordMap.put("continue", TokenKind.CONTINUE); + keywordMap.put("def", TokenKind.DEF); + keywordMap.put("del", TokenKind.DEL); + keywordMap.put("elif", TokenKind.ELIF); + keywordMap.put("else", TokenKind.ELSE); + keywordMap.put("except", TokenKind.EXCEPT); + keywordMap.put("finally", TokenKind.FINALLY); + keywordMap.put("for", TokenKind.FOR); + keywordMap.put("from", TokenKind.FROM); + keywordMap.put("global", TokenKind.GLOBAL); + keywordMap.put("if", TokenKind.IF); + keywordMap.put("import", TokenKind.IMPORT); + keywordMap.put("in", TokenKind.IN); + keywordMap.put("is", TokenKind.IS); + keywordMap.put("lambda", TokenKind.LAMBDA); + keywordMap.put("load", TokenKind.LOAD); + keywordMap.put("nonlocal", TokenKind.NONLOCAL); + keywordMap.put("not", TokenKind.NOT); + keywordMap.put("or", TokenKind.OR); + keywordMap.put("pass", TokenKind.PASS); + keywordMap.put("raise", TokenKind.RAISE); + keywordMap.put("return", TokenKind.RETURN); + keywordMap.put("try", TokenKind.TRY); + keywordMap.put("while", TokenKind.WHILE); + keywordMap.put("with", TokenKind.WITH); + keywordMap.put("yield", TokenKind.YIELD); + } + + /** + * Scans an identifier or keyword. + * + *

ON ENTRY: 'pos' is 1 + the index of the first char in the identifier. + * ON EXIT: 'pos' is 1 + the index of the last char in the identifier. + */ + private void identifierOrKeyword() { + int oldPos = pos - 1; + String id = identInterner.intern(scanIdentifier()); + TokenKind kind = keywordMap.get(id); + if (kind == null) { + setToken(TokenKind.IDENTIFIER, oldPos, pos); + // setValue allocates a new String for the raw text, but it's not retained so we don't bother + // interning it. + setValue(id); + } else { + setToken(kind, oldPos, pos); + } + } + + private String scanIdentifier() { + // Keep consistent with Identifier.isValid. + // TODO(laurentlb): Handle Unicode letters. + int oldPos = pos - 1; + while (pos < buffer.length) { + switch (buffer[pos]) { + case '_': + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': + case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': + case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': + case 's': case 't': case 'u': case 'v': case 'w': case 'x': + case 'y': case 'z': + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': + case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': + case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': + case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': + case 'Y': case 'Z': + case '0': case '1': case '2': case '3': case '4': case '5': + case '6': case '7': case '8': case '9': + pos++; + break; + default: + return bufferSlice(oldPos, pos); + } + } + return bufferSlice(oldPos, pos); + } + + /** + * Tokenizes a two-char operator. + * @return true if it tokenized an operator + */ + private boolean tokenizeTwoChars() { + if (pos + 2 >= buffer.length) { + return false; + } + char c1 = buffer[pos]; + char c2 = buffer[pos + 1]; + TokenKind tok = null; + if (c2 == '=') { + tok = EQUAL_TOKENS.get(c1); + } else if (c2 == '*' && c1 == '*') { + tok = TokenKind.STAR_STAR; + } + if (tok == null) { + return false; + } else { + setToken(tok, pos, pos + 2); + return true; + } + } + + // Returns the ith unconsumed char, or -1 for EOF. + private int peek(int i) { + return pos + i < buffer.length ? buffer[pos + i] : -1; + } + + // Consumes a char and returns the next unconsumed char, or -1 for EOF. + private int next() { + pos++; + return peek(0); + } + + /** + * Performs tokenization of the character buffer of file contents provided to the constructor. At + * least one token will be added to the tokens queue. + */ + private void tokenize() { + if (checkIndentation) { + checkIndentation = false; + computeIndentation(); + } + + // Return saved indentation tokens. + if (dents != 0) { + if (dents < 0) { + dents++; + setToken(TokenKind.OUTDENT, pos - 1, pos); + } else { + dents--; + setToken(TokenKind.INDENT, pos - 1, pos); + } + return; + } + + // TODO(adonovan): cleanup: replace break after setToken with return, + // and eliminate null-check of this.kind. + kind = null; + while (pos < buffer.length) { + if (tokenizeTwoChars()) { + pos += 2; + return; + } + char c = buffer[pos]; + pos++; + switch (c) { + case '{': + setToken(TokenKind.LBRACE, pos - 1, pos); + openParenStackDepth++; + break; + case '}': + setToken(TokenKind.RBRACE, pos - 1, pos); + popParen(); + break; + case '(': + setToken(TokenKind.LPAREN, pos - 1, pos); + openParenStackDepth++; + break; + case ')': + setToken(TokenKind.RPAREN, pos - 1, pos); + popParen(); + break; + case '[': + setToken(TokenKind.LBRACKET, pos - 1, pos); + openParenStackDepth++; + break; + case ']': + setToken(TokenKind.RBRACKET, pos - 1, pos); + popParen(); + break; + case '>': + if (peek(0) == '>' && peek(1) == '=') { + setToken(TokenKind.GREATER_GREATER_EQUALS, pos - 1, pos + 2); + pos += 2; + } else if (peek(0) == '>') { + setToken(TokenKind.GREATER_GREATER, pos - 1, pos + 1); + pos += 1; + } else { + setToken(TokenKind.GREATER, pos - 1, pos); + } + break; + case '<': + if (peek(0) == '<' && peek(1) == '=') { + setToken(TokenKind.LESS_LESS_EQUALS, pos - 1, pos + 2); + pos += 2; + } else if (peek(0) == '<') { + setToken(TokenKind.LESS_LESS, pos - 1, pos + 1); + pos += 1; + } else { + setToken(TokenKind.LESS, pos - 1, pos); + } + break; + case ':': + setToken(TokenKind.COLON, pos - 1, pos); + break; + case ',': + setToken(TokenKind.COMMA, pos - 1, pos); + break; + case '+': + setToken(TokenKind.PLUS, pos - 1, pos); + break; + case '-': + setToken(TokenKind.MINUS, pos - 1, pos); + break; + case '|': + setToken(TokenKind.PIPE, pos - 1, pos); + break; + case '=': + setToken(TokenKind.EQUALS, pos - 1, pos); + break; + case '%': + setToken(TokenKind.PERCENT, pos - 1, pos); + break; + case '~': + setToken(TokenKind.TILDE, pos - 1, pos); + break; + case '&': + setToken(TokenKind.AMPERSAND, pos - 1, pos); + break; + case '^': + setToken(TokenKind.CARET, pos - 1, pos); + break; + case '/': + if (peek(0) == '/' && peek(1) == '=') { + setToken(TokenKind.SLASH_SLASH_EQUALS, pos - 1, pos + 2); + pos += 2; + } else if (peek(0) == '/') { + setToken(TokenKind.SLASH_SLASH, pos - 1, pos + 1); + pos += 1; + } else { + // /= is handled by tokenizeTwoChars. + setToken(TokenKind.SLASH, pos - 1, pos); + } + break; + case ';': + setToken(TokenKind.SEMI, pos - 1, pos); + break; + case '*': + setToken(TokenKind.STAR, pos - 1, pos); + break; + case ' ': + case '\t': + case '\r': + /* ignore */ + break; + case '\\': + // Backslash character is valid only at the end of a line (or in a string) + if (peek(0) == '\n') { + pos += 1; // skip the end of line character + } else if (peek(0) == '\r' && peek(1) == '\n') { + pos += 2; // skip the CRLF at the end of line + } else { + setToken(TokenKind.ILLEGAL, pos - 1, pos); + setValue(Character.toString(c)); + } + break; + case '\n': + newline(); + break; + case '#': + int oldPos = pos - 1; + while (pos < buffer.length) { + c = buffer[pos]; + if (c == '\n') { + break; + } else { + pos++; + } + } + addComment(oldPos, pos); + break; + case '\'': + case '\"': + stringLiteral(c, false); + break; + default: + // detect raw strings, e.g. r"str" + if (c == 'r') { + int c0 = peek(0); + if (c0 == '\'' || c0 == '\"') { + pos++; + stringLiteral((char) c0, true); + break; + } + } + + // int or float literal, or dot + if (c == '.' || isdigit(c)) { + pos--; // unconsume + scanNumberOrDot(c); + break; + } + + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') { + identifierOrKeyword(); + } else { + error("invalid character: '" + c + "'", pos - 1); + } + break; + } // switch + if (kind != null) { // stop here if we scanned a token + return; + } + } // while + + if (indentStack.size() > 1) { // top of stack is always zero + setToken(TokenKind.NEWLINE, pos - 1, pos); + while (indentStack.size() > 1) { + indentStack.pop(); + dents--; + } + return; + } + + setToken(TokenKind.EOF, pos, pos); + } + + // Scans a number (INT or FLOAT) or DOT. + // Precondition: c == peek(0) (a dot or digit) + // + // TODO(adonovan): make this the precondition for all scan functions; + // currenly most assume their argument c has been consumed already. + private void scanNumberOrDot(int c) { + int start = this.pos; + boolean fraction = false; + boolean exponent = false; + + if (c == '.') { + // dot or start of fraction + if (!isdigit(peek(1))) { + pos++; // consume '.' + setToken(TokenKind.DOT, start, pos); + return; + } + fraction = true; + + } else if (c == '0') { + // hex, octal, binary or float + c = next(); + if (c == '.') { + fraction = true; + + } else if (c == 'x' || c == 'X') { + // hex + c = next(); + if (!isxdigit(c)) { + error("invalid hex literal", start); + } + while (isxdigit(c)) { + c = next(); + } + + } else if (c == 'o' || c == 'O') { + // octal + c = next(); + while (isdigit(c)) { + c = next(); + } + + } else if (c == 'b' || c == 'B') { + // binary + c = next(); + if (!isbdigit(c)) { + error("invalid binary literal", start); + } + while (isbdigit(c)) { + c = next(); + } + + } else { + // "0" or float or obsolete octal "0755" + while (isdigit(c)) { + c = next(); + } + if (c == '.') { + fraction = true; + } else if (c == 'e' || c == 'E') { + exponent = true; + } + } + + } else { + // decimal + while (isdigit(c)) { + c = next(); + } + if (c == '.') { + fraction = true; + } else if (c == 'e' || c == 'E') { + exponent = true; + } + } + + if (fraction) { + c = next(); // consume '.' + while (isdigit(c)) { + c = next(); + } + + if (c == 'e' || c == 'E') { + exponent = true; + } + } + + if (exponent) { + c = next(); // consume [eE] + if (c == '+' || c == '-') { + c = next(); + } + while (isdigit(c)) { + c = next(); + } + } + + // float? + if (fraction || exponent) { + setToken(TokenKind.FLOAT, start, pos); + double value = 0.0; + try { + value = Double.parseDouble(bufferSlice(start, pos)); + if (!Double.isFinite(value)) { + error("floating-point literal too large", start); + } + } catch (NumberFormatException ex) { + error("invalid float literal", start); + } + setValue(value); + return; + } + + // int + setToken(TokenKind.INT, start, pos); + String literal = bufferSlice(start, pos); + Number value = 0; + try { + value = IntLiteral.scan(literal); + } catch (NumberFormatException ex) { + error(ex.getMessage(), start); + } + setValue(value); + } + + private static boolean isdigit(int c) { + return '0' <= c && c <= '9'; + } + + private static boolean isxdigit(int c) { + return isdigit(c) || ('A' <= c && c <= 'F') || ('a' <= c && c <= 'f'); + } + + private static boolean isbdigit(int c) { + return c == '0' || c == '1'; + } + + /* + * Returns a string containing the part of the source buffer beginning at offset {@code start} and + * ending immediately before offset {@code end} (so the length of the resulting string is {@code + * end - start}). + */ + String bufferSlice(int start, int end) { + return new String(this.buffer, start, end - start); + } + + // TODO(adonovan): don't retain comments unconditionally. + private void addComment(int start, int end) { + String content = bufferSlice(start, end); + comments.add(new Comment(locs, start, content)); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/ListExpression.java b/third_party/bazel/main/java/net/starlark/java/syntax/ListExpression.java new file mode 100644 index 000000000..1eee175c6 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/ListExpression.java @@ -0,0 +1,106 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import com.google.common.base.Preconditions; +import java.util.List; + +/** Syntax node for list and tuple expressions. */ +public final class ListExpression extends Expression { + + // TODO(adonovan): split class into {List,Tuple}Expression, as a tuple may have no parens. + // Materialize all source-level expressions as a separate ParenExpression so that we can roundtrip + // faithfully. + + private final boolean isTuple; + private final int lbracketOffset; // -1 => unparenthesized non-empty tuple + private final List elements; + private final int rbracketOffset; // -1 => unparenthesized non-empty tuple + + ListExpression( + FileLocations locs, + boolean isTuple, + int lbracketOffset, + List elements, + int rbracketOffset) { + super(locs, Kind.LIST_EXPR); + // An unparenthesized tuple must be non-empty. + Preconditions.checkArgument( + !elements.isEmpty() || (lbracketOffset >= 0 && rbracketOffset >= 0)); + this.lbracketOffset = lbracketOffset; + this.isTuple = isTuple; + this.elements = elements; + this.rbracketOffset = rbracketOffset; + } + + public List getElements() { + return elements; + } + + /** Reports whether this is a tuple expression. */ + public boolean isTuple() { + return isTuple; + } + + @Override + public int getStartOffset() { + return lbracketOffset < 0 ? elements.get(0).getStartOffset() : lbracketOffset; + } + + @Override + public int getEndOffset() { + // Unlike Python, trailing commas are not allowed in unparenthesized tuples. + return rbracketOffset < 0 + ? elements.get(elements.size() - 1).getEndOffset() + : rbracketOffset + 1; + } + + @Override + public String toString() { + // Print [a, b, c, ...] up to a maximum of 4 elements or 32 chars. + StringBuilder buf = new StringBuilder(); + buf.append(isTuple() ? '(' : '['); + appendNodes(buf, elements); + if (isTuple() && elements.size() == 1) { + buf.append(','); + } + buf.append(isTuple() ? ')' : ']'); + return buf.toString(); + } + + // Appends elements to buf, comma-separated, abbreviating if they are numerous or long. + // (Also used by CallExpression.) + static void appendNodes(StringBuilder buf, List elements) { + int n = elements.size(); + for (int i = 0; i < n; i++) { + if (i > 0) { + buf.append(", "); + } + int mark = buf.length(); + buf.append(elements.get(i)); + // Abbreviate, dropping this element, if we exceed 32 chars, + // or 4 elements (with more elements following). + if (buf.length() >= 32 || (i == 4 && i + 1 < n)) { + buf.setLength(mark); + buf.append(String.format("+%d more", n - i)); + break; + } + } + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/LoadStatement.java b/third_party/bazel/main/java/net/starlark/java/syntax/LoadStatement.java new file mode 100644 index 000000000..0b8d90037 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/LoadStatement.java @@ -0,0 +1,85 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import com.google.common.collect.ImmutableList; + +/** Syntax node for a load statement. */ +public final class LoadStatement extends Statement { + + /** + * Binding represents a binding in a load statement. load("...", local = "orig") + * + *

If there's no alias, a single Identifier can be used for both local and orig. + * TODO(adonovan): don't do that; be faithful to source. + */ + public static final class Binding { + private final Identifier local; + private final Identifier orig; + + public Identifier getLocalName() { + return local; + } + + public Identifier getOriginalName() { + return orig; + } + + Binding(Identifier localName, Identifier originalName) { + this.local = localName; + this.orig = originalName; + } + } + + private final int loadOffset; + private final StringLiteral module; + private final ImmutableList bindings; + private final int rparenOffset; + + LoadStatement( + FileLocations locs, + int loadOffset, + StringLiteral module, + ImmutableList bindings, + int rparenOffset) { + super(locs, Kind.LOAD); + this.loadOffset = loadOffset; + this.module = module; + this.bindings = bindings; + this.rparenOffset = rparenOffset; + } + + public ImmutableList getBindings() { + return bindings; + } + + public StringLiteral getImport() { + return module; + } + + @Override + public int getStartOffset() { + return loadOffset; + } + + @Override + public int getEndOffset() { + return rparenOffset + 1; + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Location.java b/third_party/bazel/main/java/net/starlark/java/syntax/Location.java new file mode 100644 index 000000000..01577b5f3 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/Location.java @@ -0,0 +1,115 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.syntax; + +import com.google.common.base.Preconditions; +import javax.annotation.concurrent.Immutable; + +/** + * A Location denotes a position within a Starlark file. + * + *

A location is a triple {@code (file, line, column)}, where {@code file} is the apparent name + * of the file, {@code line} is the optional 1-based line number, and {@code column} is the optional + * 1-based column number measured in UTF-16 code units. If the column is zero it is not displayed. + * If the line number is also zero, it too is not displayed; in this case, the location denotes the + * file as a whole. + */ +@Immutable +public final class Location implements Comparable { + + private final String file; + private final int line; + private final int column; + + public Location(String file, int line, int column) { + this.file = Preconditions.checkNotNull(file); + this.line = line; + this.column = column; + } + + /** Returns the name of the file containing this location. */ + public String file() { + return file; + } + + /** Returns the line number of this location. */ + public int line() { + return line; + } + + /** Returns the column number of this location. */ + public int column() { + return column; + } + + /** + * Returns a Location for the given file, line and column. If {@code column} is non-zero, {@code + * line} too must be non-zero. + */ + public static Location fromFileLineColumn(String file, int line, int column) { + Preconditions.checkArgument(line != 0 || column == 0, "non-zero column but no line number"); + return new Location(file, line, column); + } + + /** Returns a Location for the file as a whole. */ + public static Location fromFile(String file) { + return new Location(file, 0, 0); + } + + /** + * Formats the location as {@code "file:line:col"}. If the column is zero, it is omitted. If the + * line is also zero, it too is omitted. + */ + @Override + public String toString() { + StringBuilder buf = new StringBuilder(); + buf.append(file); + if (line != 0) { + buf.append(':').append(line); + if (column != 0) { + buf.append(':').append(column); + } + } + return buf.toString(); + } + + /** Returns a three-valued lexicographical comparison of two Locations. */ + @Override + public int compareTo(Location that) { + int cmp = this.file().compareTo(that.file()); + if (cmp != 0) { + return cmp; + } + return Long.compare( + ((long) this.line << 32) | this.column, ((long) that.line << 32) | that.column); + } + + @Override + public int hashCode() { + return 97 * file.hashCode() + 37 * line + column; + } + + @Override + public boolean equals(Object that) { + return this == that + || (that instanceof Location + && this.file.equals(((Location) that).file) + && this.line == ((Location) that).line + && this.column == ((Location) that).column); + } + + /** A location for built-in functions. */ + public static final Location BUILTIN = fromFile(""); +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Node.java b/third_party/bazel/main/java/net/starlark/java/syntax/Node.java new file mode 100644 index 000000000..3cd7afad2 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/Node.java @@ -0,0 +1,128 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.syntax; + +import com.google.common.base.Preconditions; + +/** + * A Node is a node in a Starlark syntax tree. + * + *

Nodes may be constructed only by the parser. + * + *

The syntax tree records the offsets within the file of all salient tokens, such as brackets + * that mark the beginning or end of an expression, or operators whose position may be needed in a + * run-time error message. The start and end offsets of each Node are computed inductively from + * their tokens and subexpressions. Offsets are converted to Locations on demand in methods such as + * {@link #getStartLocation}. + */ +public abstract class Node { + + // Use these typical node distributions in Bazel files + // as a rough guide for optimization decisions. + // BUILD files are much more numerous than .bzl files, + // and typically larger. + // + // Large BUILD file: + // 49 % StringLiteral + // 17 % Identifier + // 12 % Argument.Keyword + // 9 % ListExpression + // 4 % CallExpression + // 3.5% ExpressionStatement + // 3.1% Comment + // 1.2% Argument.Positional + // 1.8% all others + // + // Large .bzl logic file: + // 42 % Identifier + // 12 % DotExpression + // 7.1% StringLiteral + // 6.7% Argument.Keyword + // 6.7% CallExpression + // 4.6% Argument.Positional + // 3.1% Comment + // 2.4% ListExpression + // 2.4% ExpressionStatement + // 2.2% AssignmentStatement + // 1.9% DictExpression.Entry + // 1.9% BinaryOperatorExpression + // 1.0% Comprehension + // 6 % all others + + // The FileLocations table holds the file name and a compressed + // mapping from token char offsets to Locations. + // It is shared by all nodes from the same file. + final FileLocations locs; + + Node(FileLocations locs) { + this.locs = Preconditions.checkNotNull(locs); + } + + /** + * Returns the node's start offset, as a char index (zero-based count of UTF-16 codes) from the + * start of the file. + */ + public abstract int getStartOffset(); + + /** Returns the location of the start of this syntax node. */ + public final Location getStartLocation() { + return locs.getLocation(getStartOffset()); + } + + /** Returns the char offset of the source position immediately after this syntax node. */ + public abstract int getEndOffset(); + + /** Returns the location of the source position immediately after this syntax node. */ + public final Location getEndLocation() { + return locs.getLocation(getEndOffset()); + } + + /** + * Returns a pretty-printed representation of this syntax tree. + * + *

This function returns the canonical source code corresponding to a syntax tree. Generally, + * the output can be round-tripped: pretty-printing a syntax tree then parsing the result should + * yield an equivalent syntax tree. + * + *

The pretty-printed form of a syntax tree may be used as a proxy for equality in tests. + * However, different trees may have the same printed form. In particular, {@link StarlarkFile} + * includes comments that are not reflected in the string. + */ + public final String prettyPrint() { + StringBuilder buf = new StringBuilder(); + new NodePrinter(buf).printNode(this); + return buf.toString(); + } + + /** + * Print the syntax node in a form useful for debugging. + * + *

The output is not precisely specified; use {@link #prettyPrint()} if you need more stable + * and complete information. For instance, this function may omit child statements of compound + * statements, or parentheses around some expressions. It may also abbreviate large list literals. + */ + @Override + public String toString() { + return prettyPrint(); // default behavior, overridden in several subclasses + } + + /** + * Implements the double dispatch by calling into the node specific visit method of + * the {@link NodeVisitor} + * + * @param visitor the {@link NodeVisitor} instance to dispatch to. + */ + public abstract void accept(NodeVisitor visitor); +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/NodePrinter.java b/third_party/bazel/main/java/net/starlark/java/syntax/NodePrinter.java new file mode 100644 index 000000000..f1691f448 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/NodePrinter.java @@ -0,0 +1,482 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import java.util.List; + +/** A pretty-printer for Starlark syntax trees. */ +final class NodePrinter { + + private final StringBuilder buf; + private int indent; + + NodePrinter(StringBuilder buf) { + this.buf = buf; + } + + // Constructor exposed to legacy tests. + // TODO(adonovan): rewrite the tests not to care about the indent parameter. + NodePrinter(StringBuilder buf, int indent) { + this.buf = buf; + this.indent = indent; + } + + // Main entry point for an arbitrary node. + // Called by Node.prettyPrint. + void printNode(Node n) { + if (n instanceof Expression) { + printExpr((Expression) n); + + } else if (n instanceof Statement) { + printStmt((Statement) n); + + } else if (n instanceof StarlarkFile file) { + // Only statements are printed, not comments. + for (Statement stmt : file.getStatements()) { + printStmt(stmt); + } + + } else if (n instanceof Comment comment) { + // We can't really print comments in the right place anyway, + // due to how their relative order is lost in the representation + // of StarlarkFile. So don't bother word-wrapping and just print + // it on a single line. + printIndent(); + buf.append(comment.getText()); + + } else if (n instanceof Argument) { + printArgument((Argument) n); + + } else if (n instanceof Parameter) { + printParameter((Parameter) n); + + } else if (n instanceof DictExpression.Entry) { + printDictEntry((DictExpression.Entry) n); + + } else { + throw new IllegalArgumentException("unexpected: " + n.getClass()); + } + } + + private void printSuite(List statements) { + // A suite is non-empty; pass statements are explicit. + indent++; + for (Statement stmt : statements) { + printStmt(stmt); + } + indent--; + } + + private void printIndent() { + for (int i = 0; i < indent; i++) { + buf.append(" "); + } + } + + private void printArgument(Argument arg) { + if (arg instanceof Argument.Positional) { + // nop + } else if (arg instanceof Argument.Keyword) { + buf.append(((Argument.Keyword) arg).getIdentifier().getName()); + buf.append(" = "); + } else if (arg instanceof Argument.Star) { + buf.append('*'); + } else if (arg instanceof Argument.StarStar) { + buf.append("**"); + } + printExpr(arg.getValue()); + } + + private void printParameter(Parameter param) { + if (param instanceof Parameter.Mandatory) { + buf.append(param.getName()); + } else if (param instanceof Parameter.Optional) { + buf.append(param.getName()); + buf.append('='); + printExpr(param.getDefaultValue()); + } else if (param instanceof Parameter.Star) { + buf.append('*'); + if (param.getName() != null) { + buf.append(param.getName()); + } + } else if (param instanceof Parameter.StarStar) { + buf.append("**"); + buf.append(param.getName()); + } + } + + private void printDictEntry(DictExpression.Entry e) { + printExpr(e.getKey()); + buf.append(": "); + printExpr(e.getValue()); + } + + // Appends "def f(a, ..., z):" to the buf. + // Also used by DefStatement.toString. + void printDefSignature(DefStatement def) { + buf.append("def "); + printExpr(def.getIdentifier()); + buf.append('('); + String sep = ""; + for (Parameter param : def.getParameters()) { + buf.append(sep); + printParameter(param); + sep = ", "; + } + buf.append("):"); + } + + private void printStmt(Statement s) { + printIndent(); + + switch (s.kind()) { + case ASSIGNMENT: + { + AssignmentStatement stmt = (AssignmentStatement) s; + printExpr(stmt.getLHS()); + buf.append(' '); + if (stmt.isAugmented()) { + buf.append(stmt.getOperator()); + } + buf.append("= "); + printExpr(stmt.getRHS()); + buf.append('\n'); + break; + } + + case EXPRESSION: + { + ExpressionStatement stmt = (ExpressionStatement) s; + printExpr(stmt.getExpression()); + buf.append('\n'); + break; + } + + case FLOW: + { + FlowStatement stmt = (FlowStatement) s; + buf.append(stmt.getFlowKind()).append('\n'); + break; + } + + case FOR: + { + ForStatement stmt = (ForStatement) s; + buf.append("for "); + printExpr(stmt.getVars()); + buf.append(" in "); + printExpr(stmt.getCollection()); + buf.append(":\n"); + printSuite(stmt.getBody()); + break; + } + + case DEF: + { + DefStatement stmt = (DefStatement) s; + printDefSignature(stmt); + buf.append('\n'); + printSuite(stmt.getBody()); + break; + } + + case IF: + { + IfStatement stmt = (IfStatement) s; + buf.append(stmt.isElif() ? "elif " : "if "); + printExpr(stmt.getCondition()); + buf.append(":\n"); + printSuite(stmt.getThenBlock()); + List elseBlock = stmt.getElseBlock(); + if (elseBlock != null) { + if (elseBlock.size() == 1 + && elseBlock.get(0) instanceof IfStatement + && ((IfStatement) elseBlock.get(0)).isElif()) { + printStmt(elseBlock.get(0)); + } else { + printIndent(); + buf.append("else:\n"); + printSuite(elseBlock); + } + } + break; + } + + case LOAD: + { + LoadStatement stmt = (LoadStatement) s; + buf.append("load("); + printExpr(stmt.getImport()); + for (LoadStatement.Binding binding : stmt.getBindings()) { + buf.append(", "); + Identifier local = binding.getLocalName(); + String origName = binding.getOriginalName().getName(); + if (origName.equals(local.getName())) { + buf.append('"'); + printExpr(local); + buf.append('"'); + } else { + printExpr(local); + buf.append("=\""); + buf.append(origName); + buf.append('"'); + } + } + buf.append(")\n"); + break; + } + + case RETURN: + { + ReturnStatement stmt = (ReturnStatement) s; + buf.append("return"); + if (stmt.getResult() != null) { + buf.append(' '); + printExpr(stmt.getResult()); + } + buf.append('\n'); + break; + } + } + } + + private void printExpr(Expression expr) { + switch (expr.kind()) { + case BINARY_OPERATOR: + { + BinaryOperatorExpression binop = (BinaryOperatorExpression) expr; + // TODO(bazel-team): retain parentheses in the syntax tree so we needn't + // conservatively emit them here. + buf.append('('); + printExpr(binop.getX()); + buf.append(' '); + buf.append(binop.getOperator()); + buf.append(' '); + printExpr(binop.getY()); + buf.append(')'); + break; + } + + case COMPREHENSION: + { + Comprehension comp = (Comprehension) expr; + buf.append(comp.isDict() ? '{' : '['); + printNode(comp.getBody()); // Expression or DictExpression.Entry + for (Comprehension.Clause clause : comp.getClauses()) { + buf.append(' '); + if (clause instanceof Comprehension.For forClause) { + buf.append("for "); + printExpr(forClause.getVars()); + buf.append(" in "); + printExpr(forClause.getIterable()); + } else { + Comprehension.If ifClause = (Comprehension.If) clause; + buf.append("if "); + printExpr(ifClause.getCondition()); + } + } + buf.append(comp.isDict() ? '}' : ']'); + break; + } + + case CONDITIONAL: + { + ConditionalExpression cond = (ConditionalExpression) expr; + printExpr(cond.getThenCase()); + buf.append(" if "); + printExpr(cond.getCondition()); + buf.append(" else "); + printExpr(cond.getElseCase()); + break; + } + + case DICT_EXPR: + { + DictExpression dictexpr = (DictExpression) expr; + buf.append("{"); + String sep = ""; + for (DictExpression.Entry entry : dictexpr.getEntries()) { + buf.append(sep); + printDictEntry(entry); + sep = ", "; + } + buf.append("}"); + break; + } + + case DOT: + { + DotExpression dot = (DotExpression) expr; + printExpr(dot.getObject()); + buf.append('.'); + printExpr(dot.getField()); + break; + } + + case CALL: + { + CallExpression call = (CallExpression) expr; + printExpr(call.getFunction()); + buf.append('('); + String sep = ""; + for (Argument arg : call.getArguments()) { + buf.append(sep); + printArgument(arg); + sep = ", "; + } + buf.append(')'); + break; + } + + case IDENTIFIER: + buf.append(((Identifier) expr).getName()); + break; + + case INDEX: + { + IndexExpression index = (IndexExpression) expr; + printExpr(index.getObject()); + buf.append('['); + printExpr(index.getKey()); + buf.append(']'); + break; + } + + case INT_LITERAL: + { + buf.append(((IntLiteral) expr).getValue()); + break; + } + + case FLOAT_LITERAL: + { + buf.append(((FloatLiteral) expr).getValue()); + break; + } + + case LAMBDA: + { + LambdaExpression lambda = (LambdaExpression) expr; + buf.append("lambda"); + String sep = " "; + for (Parameter param : lambda.getParameters()) { + buf.append(sep); + sep = ", "; + printParameter(param); + } + buf.append(": "); + printExpr(lambda.getBody()); + break; + } + + case LIST_EXPR: + { + ListExpression list = (ListExpression) expr; + buf.append(list.isTuple() ? '(' : '['); + String sep = ""; + for (Expression e : list.getElements()) { + buf.append(sep); + printExpr(e); + sep = ", "; + } + if (list.isTuple() && list.getElements().size() == 1) { + buf.append(','); + } + buf.append(list.isTuple() ? ')' : ']'); + break; + } + + case SLICE: + { + SliceExpression slice = (SliceExpression) expr; + printExpr(slice.getObject()); + buf.append('['); + // The first separator colon is unconditional. + // The second separator appears only if step is printed. + if (slice.getStart() != null) { + printExpr(slice.getStart()); + } + buf.append(':'); + if (slice.getStop() != null) { + printExpr(slice.getStop()); + } + if (slice.getStep() != null) { + buf.append(':'); + printExpr(slice.getStep()); + } + buf.append(']'); + break; + } + + case STRING_LITERAL: + { + StringLiteral literal = (StringLiteral) expr; + String value = literal.getValue(); + + // TODO(adonovan): record the raw text of string (and integer) literals + // so that we can use the syntax tree for source modification tools. + // However, that may come with a memory cost until we start compiling + // (at which point the cost is only transient). + // For now, just simulate the behavior of repr(str). + buf.append('"'); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + switch (c) { + case '"': + buf.append("\\\""); + break; + case '\\': + buf.append("\\\\"); + break; + case '\r': + buf.append("\\r"); + break; + case '\n': + buf.append("\\n"); + break; + case '\t': + buf.append("\\t"); + break; + default: + // The Starlark spec (and lexer) are far from complete here, + // and it's hard to come up with a clean semantics for + // string escapes that serves Java (UTF-16) and Go (UTF-8). + // Clearly string literals should not contain non-printable + // characters. For now we'll continue to pretend that all + // non-printables are < 32, but this obviously false. + if (c < 32) { + buf.append(String.format("\\x%02x", (int) c)); + } else { + buf.append(c); + } + } + } + buf.append('"'); + break; + } + + case UNARY_OPERATOR: + { + UnaryOperatorExpression unop = (UnaryOperatorExpression) expr; + // TODO(bazel-team): retain parentheses in the syntax tree so we needn't + // conservatively emit them here. + buf.append(unop.getOperator() == TokenKind.NOT ? "not " : unop.getOperator().toString()); + buf.append('('); + printExpr(unop.getX()); + buf.append(')'); + break; + } + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/NodeVisitor.java b/third_party/bazel/main/java/net/starlark/java/syntax/NodeVisitor.java new file mode 100644 index 000000000..096f9506a --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/NodeVisitor.java @@ -0,0 +1,200 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import java.util.List; + +/** A visitor for visiting the nodes of a syntax tree in lexical order. */ +public class NodeVisitor { + + public void visit(Node node) { + // dispatch to the node specific method + node.accept(this); + } + + // node-specific visit methods + + // All four subclasses of Argument are handled together. + public void visit(Argument node) { + visit(node.getValue()); + } + + // All four subclasses of Parameter are handled together. + public void visit(Parameter node) { + visit(node.getIdentifier()); + if (node.getDefaultValue() != null) { + visit(node.getDefaultValue()); + } + } + + public void visit(StarlarkFile node) { + visitBlock(node.getStatements()); + visitAll(node.getComments()); + } + + public void visit(BinaryOperatorExpression node) { + visit(node.getX()); + visit(node.getY()); + } + + public void visit(CallExpression node) { + visit(node.getFunction()); + visitAll(node.getArguments()); + } + + public void visit(Identifier node) {} + + public void visit(Comprehension node) { + for (Comprehension.Clause clause : node.getClauses()) { + if (clause instanceof Comprehension.For) { + visit((Comprehension.For) clause); + } else { + visit((Comprehension.If) clause); + } + } + visit(node.getBody()); + } + + public void visit(Comprehension.For node) { + visit(node.getVars()); + visit(node.getIterable()); + } + + public void visit(Comprehension.If node) { + visit(node.getCondition()); + } + + public void visit(ForStatement node) { + visit(node.getCollection()); + visit(node.getVars()); + visitBlock(node.getBody()); + } + + public void visit(LoadStatement node) { + for (LoadStatement.Binding binding : node.getBindings()) { + visit(binding.getLocalName()); + } + } + + public void visit(ListExpression node) { + visitAll(node.getElements()); + } + + public void visit(@SuppressWarnings("unused") IntLiteral node) {} + + public void visit(@SuppressWarnings("unused") FloatLiteral node) {} + + public void visit(@SuppressWarnings("unused") StringLiteral node) {} + + public void visit(AssignmentStatement node) { + visit(node.getRHS()); + visit(node.getLHS()); + } + + public void visit(ExpressionStatement node) { + visit(node.getExpression()); + } + + public void visit(IfStatement node) { + visit(node.getCondition()); + visitBlock(node.getThenBlock()); + if (node.getElseBlock() != null) { + visitBlock(node.getElseBlock()); + } + } + + public void visit(DefStatement node) { + visit(node.getIdentifier()); + visitAll(node.getParameters()); + visitBlock(node.getBody()); + } + + public void visit(ReturnStatement node) { + if (node.getResult() != null) { + visit(node.getResult()); + } + } + + public void visit(FlowStatement node) {} + + public void visit(DictExpression node) { + visitAll(node.getEntries()); + } + + public void visit(DictExpression.Entry node) { + visit(node.getKey()); + visit(node.getValue()); + } + + public void visit(UnaryOperatorExpression node) { + visit(node.getX()); + } + + public void visit(DotExpression node) { + visit(node.getObject()); + visit(node.getField()); + } + + public void visit(IndexExpression node) { + visit(node.getObject()); + visit(node.getKey()); + } + + public void visit(LambdaExpression node) { + visitAll(node.getParameters()); + visit(node.getBody()); + } + + public void visit(SliceExpression node) { + visit(node.getObject()); + if (node.getStart() != null) { + visit(node.getStart()); + } + if (node.getStop() != null) { + visit(node.getStop()); + } + if (node.getStep() != null) { + visit(node.getStep()); + } + } + + public void visit(@SuppressWarnings("unused") Comment node) {} + + public void visit(ConditionalExpression node) { + visit(node.getCondition()); + visit(node.getThenCase()); + if (node.getElseCase() != null) { + visit(node.getElseCase()); + } + } + + // methods dealing with sequences of nodes + public void visitAll(List nodes) { + for (Node node : nodes) { + visit(node); + } + } + + /** + * Visit a sequence ("block") of statements (e.g. an if branch, for block, function block etc.) + * + * This method allows subclasses to handle statement blocks more easily, like doing an action + * after every statement in a block without having to override visit(...) for all statements. + * + * @param statements list of statements in the block + */ + public void visitBlock(List statements) { + visitAll(statements); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Parameter.java b/third_party/bazel/main/java/net/starlark/java/syntax/Parameter.java new file mode 100644 index 000000000..8b7da9b1a --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/Parameter.java @@ -0,0 +1,147 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import javax.annotation.Nullable; + +/** + * Syntax node for a parameter in a function definition. + * + *

Parameters may be of four forms, as in {@code def f(a, b=c, *args, **kwargs)}. They are + * represented by the subclasses Mandatory, Optional, Star, and StarStar. + */ +public abstract class Parameter extends Node { + + @Nullable private final Identifier id; + + private Parameter(FileLocations locs, @Nullable Identifier id) { + super(locs); + this.id = id; + } + + @Nullable + public String getName() { + return id != null ? id.getName() : null; + } + + @Nullable + public Identifier getIdentifier() { + return id; + } + + @Nullable + public Expression getDefaultValue() { + return null; + } + + /** + * Syntax node for a mandatory parameter, {@code f(id)}. It may be positional or keyword-only + * depending on its position. + */ + public static final class Mandatory extends Parameter { + Mandatory(FileLocations locs, Identifier id) { + super(locs, id); + } + + @Override + public int getStartOffset() { + return getIdentifier().getStartOffset(); + } + + @Override + public int getEndOffset() { + return getIdentifier().getEndOffset(); + } + } + + /** + * Syntax node for an optional parameter, {@code f(id=expr).}. It may be positional or + * keyword-only depending on its position. + */ + public static final class Optional extends Parameter { + + public final Expression defaultValue; + + Optional(FileLocations locs, Identifier id, @Nullable Expression defaultValue) { + super(locs, id); + this.defaultValue = defaultValue; + } + + @Override + @Nullable + public Expression getDefaultValue() { + return defaultValue; + } + + @Override + public int getStartOffset() { + return getIdentifier().getStartOffset(); + } + + @Override + public int getEndOffset() { + return getDefaultValue().getEndOffset(); + } + + @Override + public String toString() { + return getName() + "=" + defaultValue; + } + } + + /** Syntax node for a star parameter, {@code f(*id)} or or {@code f(..., *, ...)}. */ + public static final class Star extends Parameter { + private final int starOffset; + + Star(FileLocations locs, int starOffset, @Nullable Identifier id) { + super(locs, id); + this.starOffset = starOffset; + } + + @Override + public int getStartOffset() { + return starOffset; + } + + @Override + public int getEndOffset() { + return getIdentifier().getEndOffset(); + } + } + + /** Syntax node for a parameter of the form {@code f(**id)}. */ + public static final class StarStar extends Parameter { + private final int starStarOffset; + + StarStar(FileLocations locs, int starStarOffset, Identifier id) { + super(locs, id); + this.starStarOffset = starStarOffset; + } + + @Override + public int getStartOffset() { + return starStarOffset; + } + + @Override + public int getEndOffset() { + return getIdentifier().getEndOffset(); + } + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Parser.java b/third_party/bazel/main/java/net/starlark/java/syntax/Parser.java new file mode 100644 index 000000000..2ddcc36ff --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/Parser.java @@ -0,0 +1,1261 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.errorprone.annotations.FormatMethod; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; + +/** Parser is a recursive-descent parser for Starlark. */ +final class Parser { + + /** Combines the parser result into a single value object. */ + static final class ParseResult { + // Maps char offsets in the file to Locations. + final FileLocations locs; + + /** The top-level statements of the parsed file. */ + final ImmutableList statements; + + /** The comments from the parsed file. */ + final ImmutableList comments; + + // Errors encountered during scanning or parsing. + // These lists are ultimately owned by StarlarkFile. + final List errors; + + private ParseResult( + FileLocations locs, + ImmutableList statements, + ImmutableList comments, + List errors) { + this.locs = locs; + // No need to copy here; when the object is created, the parser instance is just about to go + // out of scope and be garbage collected. + this.statements = Preconditions.checkNotNull(statements); + this.comments = Preconditions.checkNotNull(comments); + this.errors = errors; + } + } + + private static final EnumSet STATEMENT_TERMINATOR_SET = + EnumSet.of(TokenKind.EOF, TokenKind.NEWLINE, TokenKind.SEMI); + + private static final EnumSet LIST_TERMINATOR_SET = + EnumSet.of(TokenKind.EOF, TokenKind.RBRACKET, TokenKind.SEMI); + + private static final EnumSet DICT_TERMINATOR_SET = + EnumSet.of(TokenKind.EOF, TokenKind.RBRACE, TokenKind.SEMI); + + private static final EnumSet EXPR_LIST_TERMINATOR_SET = + EnumSet.of( + TokenKind.EOF, + TokenKind.NEWLINE, + TokenKind.EQUALS, + TokenKind.RBRACE, + TokenKind.RBRACKET, + TokenKind.RPAREN, + TokenKind.SEMI); + + private static final EnumSet EXPR_TERMINATOR_SET = + EnumSet.of( + TokenKind.COLON, + TokenKind.COMMA, + TokenKind.EOF, + TokenKind.FOR, + TokenKind.MINUS, + TokenKind.PERCENT, + TokenKind.PLUS, + TokenKind.RBRACKET, + TokenKind.RPAREN, + TokenKind.SLASH); + + /** Current lookahead token. May be mutated by the parser. */ + private final Lexer token; // token.kind is a prettier alias for lexer.kind + + private static final boolean DEBUGGING = false; + + private final Lexer lexer; + private final FileLocations locs; + private final List errors; + + // TODO(adonovan): opt: compute this by subtraction. + private static final Map augmentedAssignments = + new ImmutableMap.Builder() + .put(TokenKind.PLUS_EQUALS, TokenKind.PLUS) + .put(TokenKind.MINUS_EQUALS, TokenKind.MINUS) + .put(TokenKind.STAR_EQUALS, TokenKind.STAR) + .put(TokenKind.SLASH_EQUALS, TokenKind.SLASH) + .put(TokenKind.SLASH_SLASH_EQUALS, TokenKind.SLASH_SLASH) + .put(TokenKind.PERCENT_EQUALS, TokenKind.PERCENT) + .put(TokenKind.AMPERSAND_EQUALS, TokenKind.AMPERSAND) + .put(TokenKind.CARET_EQUALS, TokenKind.CARET) + .put(TokenKind.PIPE_EQUALS, TokenKind.PIPE) + .put(TokenKind.GREATER_GREATER_EQUALS, TokenKind.GREATER_GREATER) + .put(TokenKind.LESS_LESS_EQUALS, TokenKind.LESS_LESS) + .buildOrThrow(); + + /** + * Highest precedence goes last. Based on: + * http://docs.python.org/2/reference/expressions.html#operator-precedence + */ + private static final List> operatorPrecedence = + ImmutableList.of( + EnumSet.of(TokenKind.OR), + EnumSet.of(TokenKind.AND), + EnumSet.of(TokenKind.NOT), + EnumSet.of( + TokenKind.EQUALS_EQUALS, + TokenKind.NOT_EQUALS, + TokenKind.LESS, + TokenKind.LESS_EQUALS, + TokenKind.GREATER, + TokenKind.GREATER_EQUALS, + TokenKind.IN, + TokenKind.NOT_IN), + EnumSet.of(TokenKind.PIPE), + EnumSet.of(TokenKind.CARET), + EnumSet.of(TokenKind.AMPERSAND), + EnumSet.of(TokenKind.GREATER_GREATER, TokenKind.LESS_LESS), + EnumSet.of(TokenKind.MINUS, TokenKind.PLUS), + EnumSet.of(TokenKind.SLASH, TokenKind.SLASH_SLASH, TokenKind.STAR, TokenKind.PERCENT)); + + private int errorsCount; + private boolean recoveryMode; // stop reporting errors until next statement + + // Intern string literals, as some files contain many literals for the same string. + // + // Ideally we would move this to the lexer, where we already do interning of identifiers. However, + // the parser has a special case optimization for concatenation of string literals, which the + // lexer can't handle. + private final Map stringInterner = new HashMap<>(); + + private Parser(Lexer lexer, List errors) { + this.lexer = lexer; + this.locs = lexer.locs; + this.errors = errors; + this.token = lexer; + nextToken(); + } + + private String intern(String s) { + String prev = stringInterner.putIfAbsent(s, s); + return prev != null ? prev : s; + } + + // Returns a token's string form as used in error messages. + private static String tokenString(TokenKind kind, @Nullable Object value) { + return kind == TokenKind.STRING + ? "\"" + value + "\"" // TODO(adonovan): do proper quotation + : value == null ? kind.toString() : value.toString(); + } + + // Main entry point for parsing a file. + static ParseResult parseFile(ParserInput input, FileOptions options) { + List errors = new ArrayList<>(); + Lexer lexer = new Lexer(input, errors, options); + Parser parser = new Parser(lexer, errors); + + StarlarkFile.ParseProfiler profiler = Parser.profiler; + long profileStartNanos = profiler != null ? profiler.start() : -1; + try { + ImmutableList statements = parser.parseFileInput(); + return new ParseResult(lexer.locs, statements, lexer.getComments(), errors); + } finally { + if (profileStartNanos != -1) { + profiler.end(profileStartNanos, input.getFile()); + } + } + } + + @Nullable static StarlarkFile.ParseProfiler profiler; + + // stmt = simple_stmt + // | def_stmt + // | for_stmt + // | if_stmt + private void parseStatement(ImmutableList.Builder list) { + if (token.kind == TokenKind.DEF) { + list.add(parseDefStatement()); + } else if (token.kind == TokenKind.IF) { + list.add(parseIfStatement()); + } else if (token.kind == TokenKind.FOR) { + list.add(parseForStatement()); + } else { + parseSimpleStatement(list); + } + } + + /** Parses an expression, possibly followed by newline tokens. */ + static Expression parseExpression(ParserInput input, FileOptions options) + throws SyntaxError.Exception { + List errors = new ArrayList<>(); + Lexer lexer = new Lexer(input, errors, options); + Parser parser = new Parser(lexer, errors); + Expression result = null; + try { + result = parser.parseExpression(); + while (parser.token.kind == TokenKind.NEWLINE) { + parser.nextToken(); + } + parser.expect(TokenKind.EOF); + } catch (StackOverflowError ex) { + // See rationale at parseFile. + parser.reportError( + lexer.end, + "internal error: stack overflow while parsing Starlark expression <<%s>>. Please report" + + " the bug.\n" + + "%s", + new String(input.getContent()), + Throwables.getStackTraceAsString(ex)); + } + if (!errors.isEmpty()) { + throw new SyntaxError.Exception(errors); + } + return result; + } + + // Equivalent to 'testlist' rule in Python grammar. It can parse every kind of + // expression. In many cases, we need to use parseTest to avoid ambiguity: + // e.g. fct(x, y) vs fct((x, y)) + // + // A trailing comma is disallowed in an unparenthesized tuple. + // This prevents bugs where a one-element tuple is surprisingly created: + // e.g. foo = f(x), + private Expression parseExpression() { + Expression e = parseTest(); + if (token.kind != TokenKind.COMMA) { + return e; + } + + // unparenthesized tuple + List elems = new ArrayList<>(); + elems.add(e); + parseExprList(elems, /*trailingCommaAllowed=*/ false); + return new ListExpression(locs, /*isTuple=*/ true, -1, elems, -1); + } + + @FormatMethod + private void reportError(int offset, String format, Object... args) { + errorsCount++; + // Limit the number of reported errors to avoid spamming output. + if (errorsCount <= 5) { + Location location = locs.getLocation(offset); + errors.add(new SyntaxError(location, String.format(format, args))); + } + } + + private void syntaxError(String message) { + if (!recoveryMode) { + if (token.kind == TokenKind.INDENT) { + reportError(token.start, "indentation error"); + } else { + reportError( + token.start, "syntax error at '%s': %s", tokenString(token.kind, token.value), message); + } + recoveryMode = true; + } + } + + // Consumes the current token and returns its position, like nextToken. + // Reports a syntax error if the new token is not of the expected kind. + private int expect(TokenKind kind) { + if (token.kind != kind) { + syntaxError("expected " + kind); + } + return nextToken(); + } + + // Like expect, but stops recovery mode if the token was expected. + private int expectAndRecover(TokenKind kind) { + if (token.kind != kind) { + syntaxError("expected " + kind); + } else { + recoveryMode = false; + } + return nextToken(); + } + + // Consumes tokens past the first token belonging to terminatingTokens. + // It returns the end offset of the terminating token. + // TODO(adonovan): always used with makeErrorExpression. Combine and simplify. + private int syncPast(EnumSet terminatingTokens) { + Preconditions.checkState(terminatingTokens.contains(TokenKind.EOF)); + while (!terminatingTokens.contains(token.kind)) { + nextToken(); + } + int end = token.end; + // read past the synchronization token + nextToken(); + return end; + } + + /** + * Consume tokens until we reach the first token that has a kind that is in + * the set of terminatingTokens. + * @param terminatingTokens + * @return the end offset of the terminating token. + */ + private int syncTo(EnumSet terminatingTokens) { + // EOF must be in the set to prevent an infinite loop + Preconditions.checkState(terminatingTokens.contains(TokenKind.EOF)); + // read past the problematic token + int previous = token.end; + nextToken(); + int current = previous; + while (!terminatingTokens.contains(token.kind)) { + nextToken(); + previous = current; + current = token.end; + } + return previous; + } + + // Keywords that exist in Python and that we don't parse. + private static final EnumSet FORBIDDEN_KEYWORDS = + EnumSet.of( + TokenKind.AS, + TokenKind.ASSERT, + TokenKind.CLASS, + TokenKind.DEL, + TokenKind.EXCEPT, + TokenKind.FINALLY, + TokenKind.FROM, + TokenKind.GLOBAL, + TokenKind.IMPORT, + TokenKind.IS, + TokenKind.NONLOCAL, + TokenKind.RAISE, + TokenKind.TRY, + TokenKind.WITH, + TokenKind.WHILE, + TokenKind.YIELD); + + private void checkForbiddenKeywords() { + if (!FORBIDDEN_KEYWORDS.contains(token.kind)) { + return; + } + String error; + switch (token.kind) { + case ASSERT: error = "'assert' not supported, use 'fail' instead"; break; + case DEL: + error = "'del' not supported, use '.pop()' to delete an item from a dictionary or a list"; + break; + case IMPORT: error = "'import' not supported, use 'load' instead"; break; + case IS: error = "'is' not supported, use '==' instead"; break; + case RAISE: error = "'raise' not supported, use 'fail' instead"; break; + case TRY: error = "'try' not supported, all exceptions are fatal"; break; + case WHILE: error = "'while' not supported, use 'for' instead"; break; + default: + error = "keyword '" + token.kind + "' not supported"; + break; + } + reportError(token.start, "%s", error); + } + + private int nextToken() { + int prev = token.start; + if (token.kind != TokenKind.EOF) { + lexer.nextToken(); + } + checkForbiddenKeywords(); + // TODO(adonovan): move this to lexer so we see the first token too. + if (DEBUGGING) { + System.err.print(tokenString(token.kind, token.value)); + } + return prev; + } + + // Returns an "Identifier" whose content is the input from start to end. + private Identifier makeErrorExpression(int start, int end) { + // It's tempting to define a dedicated BadExpression type, + // but it is convenient for parseIdent to return an Identifier + // even when it fails. + return new Identifier(locs, lexer.bufferSlice(start, end), start); + } + + + // arg = IDENTIFIER '=' test + // | expr + // | *args + // | **kwargs + private Argument parseArgument() { + Expression expr; + + // parse **expr + if (token.kind == TokenKind.STAR_STAR) { + int starStarOffset = nextToken(); + expr = parseTest(); + return new Argument.StarStar(locs, starStarOffset, expr); + } + + // parse *expr + if (token.kind == TokenKind.STAR) { + int starOffset = nextToken(); + expr = parseTest(); + return new Argument.Star(locs, starOffset, expr); + } + + // IDENTIFIER or IDENTIFIER = test + expr = parseTest(); + if (expr instanceof Identifier id) { + // parse a named argument + if (token.kind == TokenKind.EQUALS) { + nextToken(); + Expression arg = parseTest(); + return new Argument.Keyword(locs, id, arg); + } + } + + // parse a positional argument + return new Argument.Positional(locs, expr); + } + + // arg = IDENTIFIER '=' test + // | IDENTIFIER + private Parameter parseParameter() { + // **kwargs + if (token.kind == TokenKind.STAR_STAR) { + int starStarOffset = nextToken(); + Identifier id = parseIdent(); + return new Parameter.StarStar(locs, starStarOffset, id); + } + + // * or *args + if (token.kind == TokenKind.STAR) { + int starOffset = nextToken(); + if (token.kind == TokenKind.IDENTIFIER) { + Identifier id = parseIdent(); + return new Parameter.Star(locs, starOffset, id); + } + return new Parameter.Star(locs, starOffset, null); + } + + // name=default + Identifier id = parseIdent(); + if (token.kind == TokenKind.EQUALS) { + nextToken(); // TODO: save token pos? + Expression expr = parseTest(); + return new Parameter.Optional(locs, id, expr); + } + + // name + return new Parameter.Mandatory(locs, id); + } + + // call_suffix = '(' arg_list? ')' + private Expression parseCallSuffix(Expression fn) { + ImmutableList args = ImmutableList.of(); + int lparenOffset = expect(TokenKind.LPAREN); + if (token.kind != TokenKind.RPAREN) { + args = parseArguments(); // (includes optional trailing comma) + } + int rparenOffset = expect(TokenKind.RPAREN); + return new CallExpression(locs, fn, locs.getLocation(lparenOffset), args, rparenOffset); + } + + // Parse a list of call arguments. + // + // arg_list = ( (arg ',')* arg ','? )? + private ImmutableList parseArguments() { + boolean seenArg = false; + ImmutableList.Builder list = ImmutableList.builder(); + while (token.kind != TokenKind.RPAREN && token.kind != TokenKind.EOF) { + if (seenArg) { + // f(expr for vars in expr) -- Python generator expression? + if (token.kind == TokenKind.FOR) { + syntaxError("Starlark does not support Python-style generator expressions"); + } + expect(TokenKind.COMMA); + // If nonempty, the list may end with a comma. + if (token.kind == TokenKind.RPAREN) { + break; + } + } + list.add(parseArgument()); + seenArg = true; + } + return list.build(); + } + + // selector_suffix = '.' IDENTIFIER + private Expression parseSelectorSuffix(Expression e) { + int dotOffset = expect(TokenKind.DOT); + if (token.kind == TokenKind.IDENTIFIER) { + Identifier id = parseIdent(); + return new DotExpression(locs, e, dotOffset, id); + } + + syntaxError("expected identifier after dot"); + syncTo(EXPR_TERMINATOR_SET); + return e; + } + + // expr_list parses a comma-separated list of expression. It assumes that the + // first expression was already parsed, so it starts with a comma. + // It is used to parse tuples and list elements. + // + // expr_list = ( ',' expr )* ','? + private void parseExprList(List list, boolean trailingCommaAllowed) { + // terminating tokens for an expression list + while (token.kind == TokenKind.COMMA) { + expect(TokenKind.COMMA); + if (EXPR_LIST_TERMINATOR_SET.contains(token.kind)) { + if (!trailingCommaAllowed) { + reportError(token.start, "Trailing comma is allowed only in parenthesized tuples."); + } + break; + } + list.add(parseTest()); + } + } + + // dict_entry_list = ( (dict_entry ',')* dict_entry ','? )? + private List parseDictEntryList() { + List list = new ArrayList<>(); + // the terminating token for a dict entry list + while (token.kind != TokenKind.RBRACE) { + list.add(parseDictEntry()); + if (token.kind == TokenKind.COMMA) { + nextToken(); + } else { + break; + } + } + return list; + } + + // dict_entry = test ':' test + private DictExpression.Entry parseDictEntry() { + Expression key = parseTest(); + int colonOffset = expect(TokenKind.COLON); + Expression value = parseTest(); + return new DictExpression.Entry(locs, key, colonOffset, value); + } + + // expr = STRING + private StringLiteral parseStringLiteral() { + Preconditions.checkState(token.kind == TokenKind.STRING); + StringLiteral literal = + new StringLiteral(locs, token.start, intern((String) token.value), token.end); + nextToken(); + if (token.kind == TokenKind.STRING) { + reportError(token.start, "Implicit string concatenation is forbidden, use the + operator"); + } + return literal; + } + + // primary = INT + // | FLOAT + // | STRING + // | IDENTIFIER + // | list_expression + // | '(' ')' // a tuple with zero elements + // | '(' expr ')' // a parenthesized expression + // | dict_expression + // | '-' primary_with_suffix + private Expression parsePrimary() { + switch (token.kind) { + case INT: + { + IntLiteral literal = + new IntLiteral(locs, token.getRaw(), token.start, (Number) token.value); + nextToken(); + return literal; + } + + case FLOAT: + { + FloatLiteral literal = + new FloatLiteral(locs, token.getRaw(), token.start, (double) token.value); + nextToken(); + return literal; + } + + case STRING: + return parseStringLiteral(); + + case IDENTIFIER: + return parseIdent(); + + case LBRACKET: // [...] + return parseListMaker(); + + case LBRACE: // {...} + return parseDictExpression(); + + case LPAREN: + { + int lparenOffset = nextToken(); + + // empty tuple: () + if (token.kind == TokenKind.RPAREN) { + int rparen = nextToken(); + return new ListExpression( + locs, /*isTuple=*/ true, lparenOffset, ImmutableList.of(), rparen); + } + + Expression e = parseTest(); + + // parenthesized expression: (e) + // TODO(adonovan): materialize paren expressions (for fidelity). + if (token.kind == TokenKind.RPAREN) { + nextToken(); + return e; + } + + // non-empty tuple: (e,) or (e, ..., e) + if (token.kind == TokenKind.COMMA) { + List elems = new ArrayList<>(); + elems.add(e); + parseExprList(elems, /*trailingCommaAllowed=*/ true); + int rparenOffset = expect(TokenKind.RPAREN); + return new ListExpression(locs, /*isTuple=*/ true, lparenOffset, elems, rparenOffset); + } + + // (expr for vars in expr) -- Python generator expression? + if (token.kind == TokenKind.FOR) { + syntaxError("Starlark does not support Python-style generator expressions"); + } + + expect(TokenKind.RPAREN); + int end = syncTo(EXPR_TERMINATOR_SET); + return makeErrorExpression(lparenOffset, end); + } + + case MINUS: + case PLUS: + case TILDE: + { + TokenKind op = token.kind; + int offset = nextToken(); + Expression x = parsePrimaryWithSuffix(); + return new UnaryOperatorExpression(locs, op, offset, x); + } + + default: + { + int start = token.start; + syntaxError("expected expression"); + int end = syncTo(EXPR_TERMINATOR_SET); + return makeErrorExpression(start, end); + } + } + } + + // primary_with_suffix = primary (selector_suffix | slice_suffix | call_suffix)* + private Expression parsePrimaryWithSuffix() { + Expression e = parsePrimary(); + while (true) { + if (token.kind == TokenKind.DOT) { + e = parseSelectorSuffix(e); + } else if (token.kind == TokenKind.LBRACKET) { + e = parseSliceSuffix(e); + } else if (token.kind == TokenKind.LPAREN) { + e = parseCallSuffix(e); + } else { + return e; + } + } + } + + // slice_suffix = '[' expr? ':' expr? ':' expr? ']' + // | '[' expr? ':' expr? ']' + // | '[' expr ']' + private Expression parseSliceSuffix(Expression e) { + int lbracketOffset = expect(TokenKind.LBRACKET); + Expression start = null; + Expression end = null; + Expression step = null; + + if (token.kind != TokenKind.COLON) { + start = parseExpression(); + + // index x[i] + if (token.kind == TokenKind.RBRACKET) { + int rbracketOffset = expect(TokenKind.RBRACKET); + return new IndexExpression(locs, e, lbracketOffset, start, rbracketOffset); + } + } + + // slice or substring x[i:j] or x[i:j:k] + expect(TokenKind.COLON); + if (token.kind != TokenKind.COLON && token.kind != TokenKind.RBRACKET) { + end = parseTest(); + } + if (token.kind == TokenKind.COLON) { + expect(TokenKind.COLON); + if (token.kind != TokenKind.RBRACKET) { + step = parseTest(); + } + } + int rbracketOffset = expect(TokenKind.RBRACKET); + return new SliceExpression(locs, e, lbracketOffset, start, end, step, rbracketOffset); + } + + // Equivalent to 'exprlist' rule in Python grammar. + // loop_variables = primary_with_suffix ( ',' primary_with_suffix )* ','? + private Expression parseForLoopVariables() { + // We cannot reuse parseExpression because it would parse the 'in' operator. + // e.g. "for i in e: pass" -> we want to parse only "i" here. + Expression e1 = parsePrimaryWithSuffix(); + if (token.kind != TokenKind.COMMA) { + return e1; + } + + // unparenthesized tuple + List elems = new ArrayList<>(); + elems.add(e1); + while (token.kind == TokenKind.COMMA) { + expect(TokenKind.COMMA); + if (EXPR_LIST_TERMINATOR_SET.contains(token.kind)) { + break; + } + elems.add(parsePrimaryWithSuffix()); + } + return new ListExpression(locs, /*isTuple=*/ true, -1, elems, -1); + } + + // comprehension_suffix = 'FOR' loop_variables 'IN' expr comprehension_suffix + // | 'IF' expr comprehension_suffix + // | ']' | '}' + private Expression parseComprehensionSuffix(int loffset, Node body, TokenKind closingBracket) { + ImmutableList.Builder clauses = ImmutableList.builder(); + while (true) { + if (token.kind == TokenKind.FOR) { + int forOffset = nextToken(); + Expression vars = parseForLoopVariables(); + expect(TokenKind.IN); + // The expression cannot be a ternary expression ('x if y else z') due to + // conflicts in Python grammar ('if' is used by the comprehension). + Expression seq = parseTest(0); + clauses.add(new Comprehension.For(locs, forOffset, vars, seq)); + } else if (token.kind == TokenKind.IF) { + int ifOffset = nextToken(); + // [x for x in li if 1, 2] # parse error + // [x for x in li if (1, 2)] # ok + Expression cond = parseTestNoCond(); + clauses.add(new Comprehension.If(locs, ifOffset, cond)); + } else if (token.kind == closingBracket) { + break; + } else { + syntaxError("expected '" + closingBracket + "', 'for' or 'if'"); + int end = syncPast(LIST_TERMINATOR_SET); + return makeErrorExpression(loffset, end); + } + } + + boolean isDict = closingBracket == TokenKind.RBRACE; + int roffset = expect(closingBracket); + return new Comprehension(locs, isDict, loffset, body, clauses.build(), roffset); + } + + // list_maker = '[' ']' + // | '[' expr ']' + // | '[' expr expr_list ']' + // | '[' expr comprehension_suffix ']' + private Expression parseListMaker() { + int lbracketOffset = expect(TokenKind.LBRACKET); + if (token.kind == TokenKind.RBRACKET) { // empty List + int rbracketOffset = nextToken(); + return new ListExpression( + locs, /*isTuple=*/ false, lbracketOffset, ImmutableList.of(), rbracketOffset); + } + + Expression expression = parseTest(); + switch (token.kind) { + case RBRACKET: + // [e], singleton list + { + int rbracketOffset = nextToken(); + return new ListExpression( + locs, + /*isTuple=*/ false, + lbracketOffset, + ImmutableList.of(expression), + rbracketOffset); + } + + case FOR: + // [e for x in y], list comprehension + return parseComprehensionSuffix(lbracketOffset, expression, TokenKind.RBRACKET); + + case COMMA: + // [e, ...], list expression + { + List elems = new ArrayList<>(); + elems.add(expression); + parseExprList(elems, /*trailingCommaAllowed=*/ true); + if (token.kind == TokenKind.RBRACKET) { + int rbracketOffset = nextToken(); + return new ListExpression( + locs, /*isTuple=*/ false, lbracketOffset, elems, rbracketOffset); + } + + expect(TokenKind.RBRACKET); + int end = syncPast(LIST_TERMINATOR_SET); + return makeErrorExpression(lbracketOffset, end); + } + + default: + { + syntaxError("expected ',', 'for' or ']'"); + int end = syncPast(LIST_TERMINATOR_SET); + return makeErrorExpression(lbracketOffset, end); + } + } + } + + // dict_expression = '{' '}' + // | '{' dict_entry_list '}' + // | '{' dict_entry comprehension_suffix '}' + private Expression parseDictExpression() { + int lbraceOffset = expect(TokenKind.LBRACE); + if (token.kind == TokenKind.RBRACE) { // empty Dict + int rbraceOffset = nextToken(); + return new DictExpression(locs, lbraceOffset, ImmutableList.of(), rbraceOffset); + } + + DictExpression.Entry entry = parseDictEntry(); + if (token.kind == TokenKind.FOR) { + // Dict comprehension + return parseComprehensionSuffix(lbraceOffset, entry, TokenKind.RBRACE); + } + + List entries = new ArrayList<>(); + entries.add(entry); + if (token.kind == TokenKind.COMMA) { + expect(TokenKind.COMMA); + entries.addAll(parseDictEntryList()); + } + if (token.kind == TokenKind.RBRACE) { + int rbraceOffset = nextToken(); + return new DictExpression(locs, lbraceOffset, entries, rbraceOffset); + } + + expect(TokenKind.RBRACE); + int end = syncPast(DICT_TERMINATOR_SET); + return makeErrorExpression(lbraceOffset, end); + } + + private Identifier parseIdent() { + if (token.kind != TokenKind.IDENTIFIER) { + int start = token.start; + int end = expect(TokenKind.IDENTIFIER); + return makeErrorExpression(start, end); + } + + String name = (String) token.value; + int offset = nextToken(); + return new Identifier(locs, name, offset); + } + + // binop_expression = binop_expression OP binop_expression + // | parsePrimaryWithSuffix + // This function takes care of precedence between operators (see operatorPrecedence for + // the order), and it assumes left-to-right associativity. + private Expression parseBinOpExpression(int prec) { + Expression x = parseTest(prec + 1); + // The loop is not strictly needed, but it prevents risks of stack overflow. Depth is + // limited to number of different precedence levels (operatorPrecedence.size()). + TokenKind lastOp = null; + for (;;) { + if (token.kind == TokenKind.NOT) { + // If NOT appears when we expect a binary operator, it must be followed by IN. + // Since the code expects every operator to be a single token, we push a NOT_IN token. + expect(TokenKind.NOT); + if (token.kind != TokenKind.IN) { + syntaxError("expected 'in'"); + } + token.kind = TokenKind.NOT_IN; + } + + TokenKind op = token.kind; + if (!operatorPrecedence.get(prec).contains(op)) { + return x; + } + + // Operator '==' and other operators of the same precedence (e.g. '<', 'in') + // are not associative. + if (lastOp != null && operatorPrecedence.get(prec).contains(TokenKind.EQUALS_EQUALS)) { + reportError( + token.start, + "Operator '%s' is not associative with operator '%s'. Use parens.", + lastOp, + op); + } + + int opOffset = nextToken(); + Expression y = parseTest(prec + 1); + x = optimizeBinOpExpression(x, op, opOffset, y); + lastOp = op; + } + } + + // Optimize binary expressions. + // string literal + string literal can be concatenated into one string literal + // so we don't have to do the expensive string concatenation at runtime. + private Expression optimizeBinOpExpression( + Expression x, TokenKind op, int opOffset, Expression y) { + if (op == TokenKind.PLUS && x instanceof StringLiteral && y instanceof StringLiteral) { + return new StringLiteral( + locs, + x.getStartOffset(), + intern(((StringLiteral) x).getValue() + ((StringLiteral) y).getValue()), + y.getEndOffset()); + } + return new BinaryOperatorExpression(locs, x, op, opOffset, y); + } + + // Parses a non-tuple expression ("test" in Python terminology). + private Expression parseTest() { + int start = token.start; + if (token.kind == TokenKind.LAMBDA) { + return parseLambda(/*allowCond=*/ true); + } + + Expression expr = parseTest(0); + if (token.kind == TokenKind.IF) { + nextToken(); + Expression condition = parseTest(0); + if (token.kind == TokenKind.ELSE) { + nextToken(); + Expression elseClause = parseTest(); + return new ConditionalExpression(locs, expr, condition, elseClause); + } else { + reportError(start, "missing else clause in conditional expression or semicolon before if"); + return expr; // Try to recover from error: drop the if and the expression after it. Ouch. + } + } + return expr; + } + + private Expression parseTest(int prec) { + if (prec >= operatorPrecedence.size()) { + return parsePrimaryWithSuffix(); + } + if (token.kind == TokenKind.NOT && operatorPrecedence.get(prec).contains(TokenKind.NOT)) { + return parseNotExpression(prec); + } + return parseBinOpExpression(prec); + } + + // parseLambda parses a lambda expression. + // The allowCond flag allows the body to be an 'a if b else c' conditional. + private LambdaExpression parseLambda(boolean allowCond) { + int lambdaOffset = expect(TokenKind.LAMBDA); + ImmutableList params = parseParameters(); + expect(TokenKind.COLON); + Expression body = allowCond ? parseTest() : parseTestNoCond(); + return new LambdaExpression(locs, lambdaOffset, params, body); + } + + // parseTestNoCond parses a single-component expression without + // consuming a trailing 'if expr else expr'. + private Expression parseTestNoCond() { + if (token.kind == TokenKind.LAMBDA) { + return parseLambda(/*allowCond=*/ false); + } + return parseTest(0); + } + + // not_expr = 'not' expr + private Expression parseNotExpression(int prec) { + int notOffset = expect(TokenKind.NOT); + Expression x = parseTest(prec); + return new UnaryOperatorExpression(locs, TokenKind.NOT, notOffset, x); + } + + // file_input = ('\n' | stmt)* EOF + private ImmutableList parseFileInput() { + ImmutableList.Builder list = ImmutableList.builder(); + try { + while (token.kind != TokenKind.EOF) { + if (token.kind == TokenKind.NEWLINE) { + expectAndRecover(TokenKind.NEWLINE); + } else if (recoveryMode) { + // If there was a parse error, we want to recover here + // before starting a new top-level statement. + syncTo(STATEMENT_TERMINATOR_SET); + recoveryMode = false; + } else { + parseStatement(list); + } + } + } catch (StackOverflowError ex) { + // JVM threads have very limited stack, and deeply nested inputs can + // easily cause the parser to consume all available stack. It is hard + // to anticipate all the possible recursions in the parser, especially + // when considering error recovery. Consider a long list of dicts: + // even if the intended parse tree has a depth of only two, + // if each dict contains a syntax error, the parser will go into recovery + // and may discard each dict's closing '}', turning a shallow tree + // into a deep one (see b/157470754). + // + // So, for robustness, the parser treats StackOverflowError as a parse + // error, exhorting the user to report a bug. + reportError( + token.end, + "internal error: stack overflow in Starlark parser. Please report the bug and include" + + " the text of %s.\n" + + "%s", + locs.file(), + Throwables.getStackTraceAsString(ex)); + } + return list.build(); + } + + // load '(' STRING (COMMA [IDENTIFIER EQUALS] STRING)+ COMMA? ')' + private Statement parseLoadStatement() { + int loadOffset = expect(TokenKind.LOAD); + expect(TokenKind.LPAREN); + if (token.kind != TokenKind.STRING) { + // error: module is not a string literal. + StringLiteral module = new StringLiteral(locs, token.start, "", token.end); + expect(TokenKind.STRING); + return new LoadStatement(locs, loadOffset, module, ImmutableList.of(), token.end); + } + + StringLiteral module = parseStringLiteral(); + if (token.kind == TokenKind.RPAREN) { + syntaxError("expected at least one symbol to load"); + return new LoadStatement(locs, loadOffset, module, ImmutableList.of(), token.end); + } + expect(TokenKind.COMMA); + + ImmutableList.Builder bindings = ImmutableList.builder(); + // At least one symbol is required. + parseLoadSymbol(bindings); + while (token.kind != TokenKind.RPAREN && token.kind != TokenKind.EOF) { + // A trailing comma is permitted after the last symbol. + expect(TokenKind.COMMA); + if (token.kind == TokenKind.RPAREN) { + break; + } + parseLoadSymbol(bindings); + } + + int rparen = expect(TokenKind.RPAREN); + return new LoadStatement(locs, loadOffset, module, bindings.build(), rparen); + } + + /** + * Parses the next symbol argument of a load statement and puts it into the output map. + * + *

The symbol is either "name" (STRING) or name = "declared" (IDENTIFIER EQUALS STRING). If no + * alias is used, "name" and "declared" will be identical. "Declared" refers to the original name + * in the Bazel file that should be loaded, while "name" will be the key of the entry in the map. + */ + private void parseLoadSymbol(ImmutableList.Builder symbols) { + if (token.kind != TokenKind.STRING && token.kind != TokenKind.IDENTIFIER) { + syntaxError("expected either a literal string or an identifier"); + return; + } + + String name = (String) token.value; + int nameOffset = token.start + (token.kind == TokenKind.STRING ? 1 : 0); + Identifier local = new Identifier(locs, name, nameOffset); + + Identifier original; + if (token.kind == TokenKind.STRING) { + // load(..., "name") + original = local; + } else { + // load(..., local = "orig") + // The name "orig" is morally an identifier but, for legacy reasons (specifically, + // a partial implementation of Starlark embedded in a Python interpreter used by + // tests of Blaze), it must be a quoted string literal. + expect(TokenKind.IDENTIFIER); + expect(TokenKind.EQUALS); + if (token.kind != TokenKind.STRING) { + syntaxError("expected string"); + return; + } + original = new Identifier(locs, (String) token.value, token.start + 1); + } + nextToken(); + symbols.add(new LoadStatement.Binding(local, original)); + } + + // simple_stmt = small_stmt (';' small_stmt)* ';'? NEWLINE + private void parseSimpleStatement(ImmutableList.Builder list) { + list.add(parseSmallStatement()); + + while (token.kind == TokenKind.SEMI) { + nextToken(); + if (token.kind == TokenKind.NEWLINE) { + break; + } + list.add(parseSmallStatement()); + } + expectAndRecover(TokenKind.NEWLINE); + } + + // small_stmt = assign_stmt + // | expr + // | load_stmt + // | return_stmt + // | BREAK | CONTINUE | PASS + // + // assign_stmt = expr ('=' | augassign) expr + // + // augassign = '+=' | '-=' | '*=' | '/=' | '%=' | '//=' | '&=' | '|=' | '^=' |'<<=' | '>>=' + private Statement parseSmallStatement() { + // return + if (token.kind == TokenKind.RETURN) { + return parseReturnStatement(); + } + + // control flow + if (token.kind == TokenKind.BREAK + || token.kind == TokenKind.CONTINUE + || token.kind == TokenKind.PASS) { + TokenKind kind = token.kind; + int offset = nextToken(); + return new FlowStatement(locs, kind, offset); + } + + // load + if (token.kind == TokenKind.LOAD) { + return parseLoadStatement(); + } + + Expression lhs = parseExpression(); + + // lhs = rhs or lhs += rhs + TokenKind op = augmentedAssignments.get(token.kind); + if (token.kind == TokenKind.EQUALS || op != null) { + int opOffset = nextToken(); + Expression rhs = parseExpression(); + // op == null for ordinary assignment. TODO(adonovan): represent as EQUALS. + return new AssignmentStatement(locs, lhs, op, opOffset, rhs); + } else { + return new ExpressionStatement(locs, lhs); + } + } + + // if_stmt = IF expr ':' suite [ELIF expr ':' suite]* [ELSE ':' suite]? + private IfStatement parseIfStatement() { + int ifOffset = expect(TokenKind.IF); + Expression cond = parseTest(); + expect(TokenKind.COLON); + ImmutableList body = parseSuite(); + IfStatement ifStmt = new IfStatement(locs, TokenKind.IF, ifOffset, cond, body); + IfStatement tail = ifStmt; + while (token.kind == TokenKind.ELIF) { + int elifOffset = expect(TokenKind.ELIF); + cond = parseTest(); + expect(TokenKind.COLON); + body = parseSuite(); + IfStatement elif = new IfStatement(locs, TokenKind.ELIF, elifOffset, cond, body); + tail.setElseBlock(ImmutableList.of(elif)); + tail = elif; + } + if (token.kind == TokenKind.ELSE) { + expect(TokenKind.ELSE); + expect(TokenKind.COLON); + body = parseSuite(); + tail.setElseBlock(body); + } + return ifStmt; + } + + // for_stmt = FOR IDENTIFIER IN expr ':' suite + private ForStatement parseForStatement() { + int forOffset = expect(TokenKind.FOR); + Expression vars = parseForLoopVariables(); + expect(TokenKind.IN); + Expression collection = parseExpression(); + expect(TokenKind.COLON); + ImmutableList body = parseSuite(); + return new ForStatement(locs, forOffset, vars, collection, body); + } + + // def_stmt = DEF IDENTIFIER '(' arguments ')' ':' suite + private DefStatement parseDefStatement() { + int defOffset = expect(TokenKind.DEF); + Identifier ident = parseIdent(); + expect(TokenKind.LPAREN); + ImmutableList params = parseParameters(); + expect(TokenKind.RPAREN); + expect(TokenKind.COLON); + ImmutableList block = parseSuite(); + return new DefStatement(locs, defOffset, ident, params, block); + } + + // Parse a list of function parameters. + // Validation of parameter ordering and uniqueness is the job of the Resolver. + private ImmutableList parseParameters() { + boolean hasParam = false; + ImmutableList.Builder list = ImmutableList.builder(); + + while (token.kind != TokenKind.RPAREN + && token.kind != TokenKind.COLON + && token.kind != TokenKind.EOF) { + if (hasParam) { + expect(TokenKind.COMMA); + // The list may end with a comma. + if (token.kind == TokenKind.RPAREN) { + break; + } + } + Parameter param = parseParameter(); + hasParam = true; + list.add(param); + } + return list.build(); + } + + // suite is typically what follows a colon (e.g. after def or for). + // suite = simple_stmt + // | NEWLINE INDENT stmt+ OUTDENT + private ImmutableList parseSuite() { + ImmutableList.Builder list = ImmutableList.builder(); + if (token.kind == TokenKind.NEWLINE) { + expect(TokenKind.NEWLINE); + if (token.kind != TokenKind.INDENT) { + reportError(token.start, "expected an indented block"); + return list.build(); + } + expect(TokenKind.INDENT); + while (token.kind != TokenKind.OUTDENT && token.kind != TokenKind.EOF) { + parseStatement(list); + } + expectAndRecover(TokenKind.OUTDENT); + } else { + parseSimpleStatement(list); + } + return list.build(); + } + + // return_stmt = RETURN [expr] + private ReturnStatement parseReturnStatement() { + int returnOffset = expect(TokenKind.RETURN); + + Expression result = null; + if (!STATEMENT_TERMINATOR_SET.contains(token.kind)) { + result = parseExpression(); + } + return new ReturnStatement(locs, returnOffset, result); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/ParserInput.java b/third_party/bazel/main/java/net/starlark/java/syntax/ParserInput.java new file mode 100644 index 000000000..33f6a750f --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/ParserInput.java @@ -0,0 +1,109 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.file.Files; +import java.nio.file.Paths; + +/** + * The apparent name and contents of a source file, for consumption by the parser. The file name + * appears in the location information in the syntax tree, and in error messages, but the Starlark + * interpreter will not attempt to open the file. However, the default behavior of {@link + * EvalException#getMessageWithStack} attempts to read the specified file when formatting a stack + * trace. + * + *

The parser consumes a stream of chars (UTF-16 codes), and the syntax positions reported by + * {@link Node#getStartOffset} and {@link Location.column} are effectively indices into a char + * array. + */ +public final class ParserInput { + + private final String file; + private final char[] content; + + private ParserInput(char[] content, String file) { + this.content = content; + this.file = Preconditions.checkNotNull(file); + } + + /** Returns the content of the input source. Callers must not modify the result. */ + char[] getContent() { + return content; + } + + /** Returns the apparent file name of the input source. */ + public String getFile() { + return file; + } + + /** + * Returns an input source that uses the name and content of the specified UTF-8-encoded text + * file. + */ + public static ParserInput readFile(String file) throws IOException { + byte[] utf8 = Files.readAllBytes(Paths.get(file)); + return fromUTF8(utf8, file); + } + + /** Returns an unnamed input source that reads from a list of strings, joined by newlines. */ + public static ParserInput fromLines(String... lines) { + return fromString(Joiner.on("\n").join(lines), ""); + } + + /** + * Returns an input source that reads from a UTF-8-encoded byte array. The caller is free to + * subsequently mutate the array. + */ + public static ParserInput fromUTF8(byte[] bytes, String file) { + CharBuffer cb = UTF_8.decode(ByteBuffer.wrap(bytes)); + char[] utf16 = new char[cb.length()]; + cb.get(utf16); + return fromCharArray(utf16, file); + } + + /** + * Returns an input source that reads from a Latin1-encoded byte array. The caller is free to + * subsequently mutate the array. + * + *

This function exists to support legacy uses of Latin1 in Bazel. Do not use Latin1 in new + * applications. (Consider this deprecated, without the fussy warnings.) + */ + public static ParserInput fromLatin1(byte[] bytes, String file) { + char[] chars = new char[bytes.length]; + for (int i = 0; i < bytes.length; i++) { + chars[i] = (char) (0xff & bytes[i]); + } + return new ParserInput(chars, file); + } + + /** Returns an input source that reads from the given string. */ + public static ParserInput fromString(String content, String file) { + return fromCharArray(content.toCharArray(), file); + } + + /** + * Returns an input source that reads from the given char array. The caller must not subsequently + * modify the array. + */ + public static ParserInput fromCharArray(char[] content, String file) { + return new ParserInput(content, file); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Program.java b/third_party/bazel/main/java/net/starlark/java/syntax/Program.java new file mode 100644 index 000000000..ed5a55630 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/Program.java @@ -0,0 +1,102 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +/** + * An opaque, executable representation of a valid Starlark program. Programs may + * [eventually---TODO(adonovan)] be efficiently serialized and deserialized without parsing and + * recompiling. + */ +public final class Program { + + private final Resolver.Function body; + private final ImmutableList loads; + private final ImmutableList loadLocations; + + private Program( + Resolver.Function body, ImmutableList loads, ImmutableList loadLocations) { + Preconditions.checkArgument( + loads.size() == loadLocations.size(), "each load must have a corresponding location"); + + // TODO(adonovan): compile here. + this.body = body; + this.loads = loads; + this.loadLocations = loadLocations; + } + + // TODO(adonovan): eliminate once Eval no longer needs access to syntax. + public Resolver.Function getResolvedFunction() { + return body; + } + + /** Returns the file name of this compiled program. */ + public String getFilename() { + return body.getLocation().file(); + } + + /** Returns the list of load strings of this compiled program, in source order. */ + public ImmutableList getLoads() { + return loads; + } + + /*** Returns the location of the ith load (see {@link #getLoads}). */ + public Location getLoadLocation(int i) { + return loadLocations.get(i); + } + + /** + * Resolves a file syntax tree in the specified environment and compiles it to a Program. This + * operation mutates the syntax tree, both by resolving identifiers and recording local variables, + * and in case of error, by appending to {@code file.errors()}. + * + * @throws SyntaxError.Exception in case of resolution error, or if the syntax tree already + * contained syntax scan/parse errors. Resolution errors are added to {@code file.errors()}. + */ + public static Program compileFile(StarlarkFile file, Resolver.Module env) + throws SyntaxError.Exception { + Resolver.resolveFile(file, env); + if (!file.ok()) { + throw new SyntaxError.Exception(file.errors()); + } + + // Extract load statements. + ImmutableList.Builder loads = ImmutableList.builder(); + ImmutableList.Builder loadLocations = ImmutableList.builder(); + for (Statement stmt : file.getStatements()) { + if (stmt instanceof LoadStatement load) { + String module = load.getImport().getValue(); + loads.add(module); + loadLocations.add(load.getImport().getLocation()); + } + } + + return new Program(file.getResolvedFunction(), loads.build(), loadLocations.build()); + } + + /** + * Resolves an expression syntax tree in the specified environment and compiles it to a Program. + * This operation mutates the syntax tree. The {@code options} must match those used when parsing + * expression. + * + * @throws SyntaxError.Exception in case of resolution error. + */ + public static Program compileExpr(Expression expr, Resolver.Module module, FileOptions options) + throws SyntaxError.Exception { + Resolver.Function body = Resolver.resolveExpr(expr, module, options); + return new Program(body, /*loads=*/ ImmutableList.of(), /*loadLocations=*/ ImmutableList.of()); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Resolver.java b/third_party/bazel/main/java/net/starlark/java/syntax/Resolver.java new file mode 100644 index 000000000..6bb279cba --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/Resolver.java @@ -0,0 +1,1103 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.errorprone.annotations.FormatMethod; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nullable; +import net.starlark.java.spelling.SpellChecker; + +/** + * The Resolver resolves each identifier in a syntax tree to its binding, and performs other + * validity checks. + * + *

When a variable is defined, it is visible in the entire block. For example, a global variable + * is visible in the entire file; a variable in a function is visible in the entire function block + * (even on the lines before its first assignment). + * + *

Resolution is a mutation of the syntax tree, as it attaches binding information to Identifier + * nodes. (In the future, it will attach additional information to functions to support lexical + * scope, and even compilation of the trees to bytecode.) Resolution errors are reported in the + * analogous manner to scan/parse errors: for a StarlarkFile, they are appended to {@code + * StarlarkFile.errors}; for an expression they are reported by an SyntaxError.Exception exception. + * It is legal to resolve a file that already contains scan/parse errors, though it may lead to + * secondary errors. + */ +public final class Resolver extends NodeVisitor { + + // TODO(adonovan): + // - use "keyword" (not "named") and "required" (not "mandatory") terminology everywhere, + // including the spec. + // - move the "no if statements at top level" check to bazel's check{Build,*}Syntax + // (that's a spec change), or put it behind a FileOptions flag (no spec change). + + /** Scope discriminates the scope of a binding: global, local, etc. */ + public enum Scope { + /** Binding is local to a function, comprehension, or file (e.g. load). */ + LOCAL, + /** Binding is non-local and occurs outside any function or comprehension. */ + GLOBAL, + /** Binding is local to a function, comprehension, or file, but shared with nested functions. */ + CELL, + /** Binding is an implicit parameter whose value is the CELL of some enclosing function. */ + FREE, + /** Binding is predeclared by the application (e.g. glob in Bazel). */ + PREDECLARED, + /** Binding is predeclared by the core (e.g. None). */ + UNIVERSAL; + + @Override + public String toString() { + return super.toString().toLowerCase(); + } + } + + /** + * A Binding is a static abstraction of a variable. The Resolver maps each Identifier to a + * Binding. + */ + public static final class Binding { + private Scope scope; + private final int index; // index within frame (LOCAL/CELL), freevars (FREE), or module (GLOBAL) + @Nullable private final Identifier first; // first binding use, if syntactic + + private Binding(Scope scope, int index, @Nullable Identifier first) { + this.scope = scope; + this.index = index; + this.first = first; + } + + /** Returns the name of this binding's identifier. */ + @Nullable + public String getName() { + return first != null ? first.getName() : null; + } + + /** Returns the scope of the binding. */ + public Scope getScope() { + return scope; + } + + /** + * Returns the index of a binding within its function's frame (LOCAL/CELL), freevars (FREE), or + * module (GLOBAL). + */ + public int getIndex() { + return index; + } + + @Override + public String toString() { + return first == null + ? scope.toString() + : String.format( + "%s[%d] %s @ %s", scope, index, first.getName(), first.getStartLocation()); + } + } + + /** A Resolver.Function records information about a resolved function. */ + public static final class Function { + + private final String name; + private final Location location; + private final ImmutableList params; + private final ImmutableList body; + private final boolean hasVarargs; + private final boolean hasKwargs; + private final int numKeywordOnlyParams; + private final ImmutableList parameterNames; + private final boolean isToplevel; + private final ImmutableList locals; + private final int[] cellIndices; + private final ImmutableList freevars; + private final ImmutableList globals; // TODO(adonovan): move to Program. + + private Function( + String name, + Location loc, + ImmutableList params, + ImmutableList body, + boolean hasVarargs, + boolean hasKwargs, + int numKeywordOnlyParams, + List locals, + List freevars, + List globals) { + this.name = name; + this.location = loc; + this.params = params; + this.body = body; + this.hasVarargs = hasVarargs; + this.hasKwargs = hasKwargs; + this.numKeywordOnlyParams = numKeywordOnlyParams; + + ImmutableList.Builder names = ImmutableList.builderWithExpectedSize(params.size()); + for (Parameter p : params) { + names.add(p.getName()); + } + this.parameterNames = names.build(); + + this.isToplevel = name.equals(""); + this.locals = ImmutableList.copyOf(locals); + this.freevars = ImmutableList.copyOf(freevars); + this.globals = ImmutableList.copyOf(globals); + + // Create an index of the locals that are cells. + int ncells = 0; + int nlocals = locals.size(); + for (int i = 0; i < nlocals; i++) { + if (locals.get(i).scope == Scope.CELL) { + ncells++; + } + } + this.cellIndices = new int[ncells]; + for (int i = 0, j = 0; i < nlocals; i++) { + if (locals.get(i).scope == Scope.CELL) { + cellIndices[j++] = i; + } + } + } + + /** + * Returns the name of the function. It may be "" for the implicit function that holds + * the top-level statements of a file, or "" for the implicit function that evaluates a + * single expression. + */ + public String getName() { + return name; + } + + /** Returns the value denoted by the function's doc string literal, or null if absent. */ + @Nullable + public String getDocumentation() { + if (getBody().isEmpty()) { + return null; + } + Statement first = getBody().get(0); + if (!(first instanceof ExpressionStatement)) { + return null; + } + Expression expr = ((ExpressionStatement) first).getExpression(); + if (!(expr instanceof StringLiteral)) { + return null; + } + return ((StringLiteral) expr).getValue(); + } + + /** Returns the function's local bindings, parameters first. */ + public ImmutableList getLocals() { + return locals; + } + + /** + * Returns the indices within {@code getLocals()} of the "cells", that is, local variables of + * thus function that are shared with nested functions. The caller must not modify the result. + */ + public int[] getCellIndices() { + return cellIndices; + } + + /** + * Returns the list of names of globals referenced by this function. The order matches the + * indices used in compiled code. + */ + public ImmutableList getGlobals() { + return globals; + } + + /** + * Returns the list of enclosing CELL or FREE bindings referenced by this function. At run time, + * these values, all of which are cells containing variables local to some enclosing function, + * will be stored in the closure. (CELL bindings in this list are local to the immediately + * enclosing function, while FREE bindings pass through one or more intermediate enclosing + * functions.) + */ + public ImmutableList getFreeVars() { + return freevars; + } + + /** Returns the location of the function's identifier. */ + public Location getLocation() { + return location; + } + + /** + * Returns the function's parameters, in "run-time order": non-keyword-only parameters, + * keyword-only parameters, {@code *args}, and finally {@code **kwargs}. A bare {@code *} + * parameter is dropped. + */ + public ImmutableList getParameters() { + return params; + } + + /** + * Returns the effective statements of the function's body. (For the implicit function created + * to evaluate a single standalone expression, this may contain a synthesized Return statement.) + */ + // TODO(adonovan): eliminate when we switch to compiler. + public ImmutableList getBody() { + return body; + } + + /** Reports whether the function has an {@code *args} parameter. */ + public boolean hasVarargs() { + return hasVarargs; + } + + /** Reports whether the function has a {@code **kwargs} parameter. */ + public boolean hasKwargs() { + return hasKwargs; + } + + /** + * Returns the number of the function's keyword-only parameters, such as {@code c} in {@code def + * f(a, *b, c, **d)} or {@code def f(a, *, c, **d)}. + */ + public int numKeywordOnlyParams() { + return numKeywordOnlyParams; + } + + /** Returns the names of the parameters. Order is as for {@link #getParameters}. */ + public ImmutableList getParameterNames() { + return parameterNames; + } + + /** + * isToplevel indicates that this is the function containing top-level statements of + * a file. + */ + // TODO(adonovan): remove this when we remove Bazel's "export" hack, + // or switch to a compiled representation of function bodies. + public boolean isToplevel() { + return isToplevel; + } + } + + /** + * A Module is a static abstraction of a Starlark module (see {@link + * net.starlark.java.eval.Module})). It describes, for the resolver and compiler, the set of + * variable names that are predeclared, either by the interpreter (UNIVERSAL) or by the + * application (PREDECLARED), plus the set of pre-defined global names (which is typically empty, + * except in a REPL or EvaluationTestCase scenario). + */ + public interface Module { + + /** + * Resolves a name to a GLOBAL, PREDECLARED, or UNIVERSAL binding. + * + * @throws Undefined if the name is not defined. + */ + Scope resolve(String name) throws Undefined; + + /** + * An Undefined exception indicates a failure to resolve a top-level name. If {@code candidates} + * is non-null, it provides the set of accessible top-level names, which, along with local + * names, will be used as candidates for spelling suggestions. + */ + final class Undefined extends Exception { + @Nullable private final Set candidates; + + public Undefined(String message, @Nullable Set candidates) { + super(message); + this.candidates = candidates; + } + } + } + + // A simple implementation of the Module for testing. + // It defines only the predeclared names---no "universal" names (e.g. None) + // or initially-defined globals (as happens in a REPL). + // Realistically, most clients will use an eval.Module. + // TODO(adonovan): move into test/ tree. + public static Module moduleWithPredeclared(String... names) { + ImmutableSet predeclared = ImmutableSet.copyOf(names); + return (name) -> { + if (predeclared.contains(name)) { + return Scope.PREDECLARED; + } + throw new Resolver.Module.Undefined( + String.format("name '%s' is not defined", name), predeclared); + }; + } + + /** + * Represents a lexical block. + * + *

Blocks should not be confused with frames. A block generally (but not always) corresponds to + * a syntactic element that may introduce variables; the variable is only accessible within the + * block (and its descendants, unless shadowed). A frame is the place where the variable's content + * will be stored, and is associated with the current enclosing function. Blocks are used to map + * an identifier to the proper variable binding, whereas frames are used to ensure each binding + * has a distinct slot of memory. + * + *

In particular, comprehension expressions have their own block but share the same underlying + * frame as their enclosing function. This means that comprehension-local variables are not + * accessible outside the comprehension, yet these variables are still stored alongside the other + * local variables of the function. + */ + private static class Block { + @Nullable private final Block parent; // enclosing block, or null for tail of list + @Nullable Node syntax; // Comprehension, DefStatement/LambdaExpression, StarlarkFile, or null + private final ArrayList frame; // accumulated locals of enclosing function + // Accumulated CELL/FREE bindings of the enclosing function that will provide + // the values for the free variables of this function; see Function.getFreeVars. + // Null for toplevel functions and expressions, which have no free variables. + @Nullable private final ArrayList freevars; + + // Bindings for names defined in this block. + // Also, as an optimization, memoized lookups of enclosing bindings. + private final Map bindings = new HashMap<>(); + + Block( + @Nullable Block parent, + @Nullable Node syntax, + ArrayList frame, + @Nullable ArrayList freevars) { + this.parent = parent; + this.syntax = syntax; + this.frame = frame; + this.freevars = freevars; + } + } + + private final List errors; + private final FileOptions options; + private final Module module; + // List whose order defines the numbering of global variables in this program. + private final List globals = new ArrayList<>(); + // A cache of PREDECLARED, UNIVERSAL, and GLOBAL bindings queried from the module. + private final Map toplevel = new HashMap<>(); + // Linked list of blocks, innermost first, for functions and comprehensions and (finally) file. + private Block locals; + private int loopCount; + + private Resolver(List errors, Module module, FileOptions options) { + this.errors = errors; + this.module = module; + this.options = options; + } + + // Formats and reports an error at the start of the specified node. + @FormatMethod + private void errorf(Node node, String format, Object... args) { + errorf(node.getStartLocation(), format, args); + } + + // Formats and reports an error at the specified location. + @FormatMethod + private void errorf(Location loc, String format, Object... args) { + errors.add(new SyntaxError(loc, String.format(format, args))); + } + + /** + * First pass: add bindings for all variables to the current block. This is done because symbols + * are sometimes used before their definition point (e.g. functions are not necessarily declared + * in order). + */ + private void createBindingsForBlock(Iterable stmts) { + for (Statement stmt : stmts) { + createBindings(stmt); + } + } + + private void createBindings(Statement stmt) { + switch (stmt.kind()) { + case ASSIGNMENT: + createBindingsForLHS(((AssignmentStatement) stmt).getLHS()); + break; + case IF: + IfStatement ifStmt = (IfStatement) stmt; + createBindingsForBlock(ifStmt.getThenBlock()); + if (ifStmt.getElseBlock() != null) { + createBindingsForBlock(ifStmt.getElseBlock()); + } + break; + case FOR: + ForStatement forStmt = (ForStatement) stmt; + createBindingsForLHS(forStmt.getVars()); + createBindingsForBlock(forStmt.getBody()); + break; + case DEF: + DefStatement def = (DefStatement) stmt; + bind(def.getIdentifier(), /*isLoad=*/ false); + break; + case LOAD: + LoadStatement load = (LoadStatement) stmt; + Set names = new HashSet<>(); + for (LoadStatement.Binding b : load.getBindings()) { + // Reject load('...', '_private'). + Identifier orig = b.getOriginalName(); + if (orig.isPrivate() && !options.allowLoadPrivateSymbols()) { + errorf(orig, "symbol '%s' is private and cannot be imported", orig.getName()); + } + + // A load statement may not bind a single name more than once, + // even if options.allowToplevelRebinding. + Identifier local = b.getLocalName(); + if (names.add(local.getName())) { + bind(local, /*isLoad=*/ true); + } else { + errorf(local, "load statement defines '%s' more than once", local.getName()); + } + } + break; + case EXPRESSION: + case FLOW: + case RETURN: + // nothing to declare + } + } + + private void createBindingsForLHS(Expression lhs) { + for (Identifier id : Identifier.boundIdentifiers(lhs)) { + bind(id, /*isLoad=*/ false); + } + } + + private void assign(Expression lhs) { + if (lhs instanceof Identifier) { + // Bindings are created by the first pass (createBindings), + // so there's nothing to do here. + } else if (lhs instanceof IndexExpression) { + visit(lhs); + } else if (lhs instanceof ListExpression) { + for (Expression elem : ((ListExpression) lhs).getElements()) { + assign(elem); + } + } else if (lhs instanceof DotExpression) { + visit(((DotExpression) lhs).getObject()); + } else { + errorf(lhs, "cannot assign to '%s'", lhs); + } + } + + @Override + public void visit(Identifier id) { + Binding bind = use(id); + if (bind != null) { + id.setBinding(bind); + return; + } + } + + @Override + public void visit(ReturnStatement node) { + if (locals.syntax instanceof StarlarkFile) { + errorf(node, "return statements must be inside a function"); + } + super.visit(node); + } + + @Override + public void visit(CallExpression node) { + // validate call arguments + boolean seenVarargs = false; + boolean seenKwargs = false; + Set keywords = null; + for (Argument arg : node.getArguments()) { + if (arg instanceof Argument.Positional) { + if (seenVarargs) { + errorf(arg, "positional argument may not follow *args"); + } else if (seenKwargs) { + errorf(arg, "positional argument may not follow **kwargs"); + } else if (keywords != null) { + errorf(arg, "positional argument may not follow keyword argument"); + } + + } else if (arg instanceof Argument.Keyword) { + String keyword = ((Argument.Keyword) arg).getName(); + if (seenVarargs) { + errorf(arg, "keyword argument %s may not follow *args", keyword); + } else if (seenKwargs) { + errorf(arg, "keyword argument %s may not follow **kwargs", keyword); + } + if (keywords == null) { + keywords = new HashSet<>(); + } + if (!keywords.add(keyword)) { + errorf(arg, "duplicate keyword argument: %s", keyword); + } + + } else if (arg instanceof Argument.Star) { + if (seenKwargs) { + errorf(arg, "*args may not follow **kwargs"); + } else if (seenVarargs) { + errorf(arg, "multiple *args not allowed"); + } + seenVarargs = true; + + } else if (arg instanceof Argument.StarStar) { + if (seenKwargs) { + errorf(arg, "multiple **kwargs not allowed"); + } + seenKwargs = true; + } + } + + super.visit(node); + } + + @Override + public void visit(ForStatement node) { + if (locals.syntax instanceof StarlarkFile) { + errorf( + node, + "for loops are not allowed at the top level. You may move it inside a function " + + "or use a comprehension, [f(x) for x in sequence]"); + } + loopCount++; + visit(node.getCollection()); + assign(node.getVars()); + visitBlock(node.getBody()); + Preconditions.checkState(loopCount > 0); + loopCount--; + } + + @Override + public void visit(LoadStatement node) { + if (!(locals.syntax instanceof StarlarkFile)) { + errorf(node, "load statement not at top level"); + } + // Skip super.visit: don't revisit local Identifier as a use. + } + + @Override + public void visit(FlowStatement node) { + if (node.getFlowKind() != TokenKind.PASS && loopCount <= 0) { + errorf(node, "%s statement must be inside a for loop", node.getFlowKind()); + } + super.visit(node); + } + + @Override + public void visit(DotExpression node) { + visit(node.getObject()); + // Do not visit the field. + } + + @Override + public void visit(Comprehension node) { + ImmutableList clauses = node.getClauses(); + + // Following Python3, the first for clause is resolved + // outside the comprehension block. All the other loops + // are resolved in the scope of their own bindings, + // permitting forward references. + Comprehension.For for0 = (Comprehension.For) clauses.get(0); + visit(for0.getIterable()); + + // A comprehension defines a distinct lexical block in the same function's frame. + // New bindings go in the frame but aren't visible to the parent block. + pushLocalBlock(node, this.locals.frame, this.locals.freevars); + + for (Comprehension.Clause clause : clauses) { + if (clause instanceof Comprehension.For forClause) { + createBindingsForLHS(forClause.getVars()); + } + } + for (int i = 0; i < clauses.size(); i++) { + Comprehension.Clause clause = clauses.get(i); + if (clause instanceof Comprehension.For forClause) { + if (i > 0) { + visit(forClause.getIterable()); + } + assign(forClause.getVars()); + } else { + Comprehension.If ifClause = (Comprehension.If) clause; + visit(ifClause.getCondition()); + } + } + visit(node.getBody()); + popLocalBlock(); + } + + @Override + public void visit(DefStatement node) { + node.setResolvedFunction( + resolveFunction( + node, + node.getIdentifier().getName(), + node.getIdentifier().getStartLocation(), + node.getParameters(), + node.getBody())); + } + + @Override + public void visit(LambdaExpression expr) { + expr.setResolvedFunction( + resolveFunction( + expr, + "lambda", + expr.getStartLocation(), + expr.getParameters(), + ImmutableList.of(ReturnStatement.make(expr.getBody())))); + } + + @Override + public void visit(IfStatement node) { + if (locals.syntax instanceof StarlarkFile) { + errorf( + node, + "if statements are not allowed at the top level. You may move it inside a function " + + "or use an if expression (x if condition else y)."); + } + super.visit(node); + } + + @Override + public void visit(AssignmentStatement node) { + visit(node.getRHS()); + + // Disallow: [e, ...] += rhs + // Other bad cases are handled in assign. + if (node.isAugmented() && node.getLHS() instanceof ListExpression) { + errorf( + node.getOperatorLocation(), + "cannot perform augmented assignment on a list or tuple expression"); + } + + assign(node.getLHS()); + } + + // Resolves a non-binding identifier to an existing binding, or null. + @Nullable + private Binding use(Identifier id) { + String name = id.getName(); + + // Locally defined in this function, comprehension, + // or file block, or an enclosing one? + Binding bind = lookupLexical(name, locals); + if (bind != null) { + return bind; + } + + // Defined at toplevel (global, predeclared, universal)? + bind = toplevel.get(name); + if (bind != null) { + return bind; + } + Scope scope; + try { + scope = module.resolve(name); + } catch (Resolver.Module.Undefined ex) { + if (!Identifier.isValid(name)) { + // If Identifier was created by Parser.makeErrorExpression, it + // contains misparsed text. Ignore ex and report an appropriate error. + errorf(id, "contains syntax errors"); + } else if (ex.candidates != null) { + // Exception provided toplevel candidates. + // Show spelling suggestions of all symbols in scope, + String suggestion = SpellChecker.didYouMean(name, getAllSymbols(ex.candidates)); + errorf(id, "%s%s", ex.getMessage(), suggestion); + } else { + errorf(id, "%s", ex.getMessage()); + } + return null; + } + switch (scope) { + case GLOBAL: + bind = new Binding(scope, globals.size(), id); + // Accumulate globals in module. + globals.add(name); + break; + case PREDECLARED: + case UNIVERSAL: + bind = new Binding(scope, 0, id); // index not used + break; + default: + throw new IllegalStateException("bad scope: " + scope); + } + toplevel.put(name, bind); + return bind; + } + + // lookupLexical finds a lexically enclosing local binding of the name, + // plumbing it through enclosing functions as needed. + private static Binding lookupLexical(String name, Block b) { + Binding bind = b.bindings.get(name); + if (bind != null) { + return bind; + } + + if (b.parent != null) { + bind = lookupLexical(name, b.parent); + if (bind != null) { + // If a local binding was found in a parent block, + // and this block is a function, then it is a free variable + // of this function and must be plumbed through. + // Add an implicit FREE binding (a hidden parameter) to this function, + // and record the outer binding that will supply its value when + // we construct the closure. + // Also, mark the outer LOCAL as a CELL: a shared, indirect local. + // (For a comprehension block there's nothing to do, + // because it's part of the same frame as the enclosing block.) + // + // This step may occur many times if the lookupLexical + // recursion returns through many functions. + if (b.syntax instanceof DefStatement || b.syntax instanceof LambdaExpression) { + Scope scope = bind.getScope(); + if (scope == Scope.LOCAL || scope == Scope.FREE || scope == Scope.CELL) { + if (scope == Scope.LOCAL) { + bind.scope = Scope.CELL; + } + int index = b.freevars.size(); + b.freevars.add(bind); + bind = new Binding(Scope.FREE, index, bind.first); + } + } + + // Memoize, to avoid duplicate free vars and repeated walks. + b.bindings.put(name, bind); + } + } + + return bind; + } + + // Common code for def, lambda. + private Function resolveFunction( + Node syntax, // DefStatement or LambdaExpression + String name, + Location loc, + ImmutableList parameters, + ImmutableList body) { + + // Resolve defaults in enclosing environment. + for (Parameter param : parameters) { + if (param instanceof Parameter.Optional) { + visit(param.getDefaultValue()); + } + } + + // Enter function block. + ArrayList frame = new ArrayList<>(); + ArrayList freevars = new ArrayList<>(); + pushLocalBlock(syntax, frame, freevars); + + // Check parameter order and convert to run-time order: + // positionals, keyword-only, *args, **kwargs. + Parameter.Star star = null; + Parameter.StarStar starStar = null; + boolean seenOptional = false; + int numKeywordOnlyParams = 0; + // TODO(adonovan): opt: when all Identifiers are resolved to bindings accumulated + // in the function, params can be a prefix of the function's array of bindings. + ImmutableList.Builder params = + ImmutableList.builderWithExpectedSize(parameters.size()); + for (Parameter param : parameters) { + if (param instanceof Parameter.Mandatory) { + // e.g. id + if (starStar != null) { + errorf( + param, + "required parameter %s may not follow **%s", + param.getName(), + starStar.getName()); + } else if (star != null) { + numKeywordOnlyParams++; + } else if (seenOptional) { + errorf( + param, + "required positional parameter %s may not follow an optional parameter", + param.getName()); + } + bindParam(params, param); + + } else if (param instanceof Parameter.Optional) { + // e.g. id = default + seenOptional = true; + if (starStar != null) { + errorf(param, "optional parameter may not follow **%s", starStar.getName()); + } else if (star != null) { + numKeywordOnlyParams++; + } + bindParam(params, param); + + } else if (param instanceof Parameter.Star) { + // * or *args + if (starStar != null) { + errorf(param, "* parameter may not follow **%s", starStar.getName()); + } else if (star != null) { + errorf(param, "multiple * parameters not allowed"); + } else { + star = (Parameter.Star) param; + } + + } else { + // **kwargs + if (starStar != null) { + errorf(param, "multiple ** parameters not allowed"); + } + starStar = (Parameter.StarStar) param; + } + } + + // * or *args + if (star != null) { + if (star.getIdentifier() != null) { + bindParam(params, star); + } else if (numKeywordOnlyParams == 0) { + errorf(star, "bare * must be followed by keyword-only parameters"); + } + } + + // **kwargs + if (starStar != null) { + bindParam(params, starStar); + } + + createBindingsForBlock(body); + visitAll(body); + popLocalBlock(); + + return new Function( + name, + loc, + params.build(), + body, + star != null && star.getIdentifier() != null, + starStar != null, + numKeywordOnlyParams, + frame, + freevars, + globals); + } + + private void bindParam(ImmutableList.Builder params, Parameter param) { + if (bind(param.getIdentifier(), /*isLoad=*/ false)) { + errorf(param, "duplicate parameter: %s", param.getName()); + } + params.add(param); + } + + /** + * Process a binding use of a name by adding a binding to the current block if not already bound, + * and associate the identifier with it. Reports whether the name was already bound in this block. + */ + private boolean bind(Identifier id, boolean isLoad) { + String name = id.getName(); + boolean isNew = false; + Binding bind; + + // TODO(adonovan): factor out bindLocal/bindGlobal cases + // and simply the condition below. + + // outside any function/comprehension, and not a (local) load? => global binding. + if (locals.syntax instanceof StarlarkFile && !(isLoad && !options.loadBindsGlobally())) { + bind = toplevel.get(name); + if (bind == null) { + // New global binding: add to module and to toplevel cache. + isNew = true; + bind = new Binding(Scope.GLOBAL, globals.size(), id); + globals.add(name); + toplevel.put(name, bind); + + // Does this new global binding conflict with a file-local load binding? + Binding prevLocal = locals.bindings.get(name); + if (prevLocal != null) { + globalLocalConflict(id, bind.scope, prevLocal); // global, local + } + + } else { + toplevelRebinding(id, bind); // global, global + } + + } else { + // Binding is local to file, function, or comprehension. + bind = locals.bindings.get(name); + if (bind == null) { + // New local binding: add to current block's bindings map, current function's frame. + // (These are distinct entities in the case where the current block is a comprehension.) + isNew = true; + bind = new Binding(Scope.LOCAL, locals.frame.size(), id); + locals.bindings.put(name, bind); + locals.frame.add(bind); + } + + if (isLoad) { + // Does this (file-local) load binding conflict with a previous one? + if (!isNew) { + toplevelRebinding(id, bind); // local, local + } + + // ...or a previous global? + Binding prev = toplevel.get(name); + if (prev != null && prev.scope == Scope.GLOBAL) { + globalLocalConflict(id, bind.scope, prev); // local, global + } + } + } + + id.setBinding(bind); + return !isNew; + } + + // Report conflicting top-level bindings of same scope, unless options.allowToplevelRebinding. + private void toplevelRebinding(Identifier id, Binding prev) { + if (!options.allowToplevelRebinding()) { + errorf(id, "'%s' redeclared at top level", id.getName()); + if (prev.first != null) { + errorf(prev.first, "'%s' previously declared here", id.getName()); + } + } + } + + // Report global/local scope conflict on top-level bindings. + private void globalLocalConflict(Identifier id, Scope scope, Binding prev) { + String newqual = scope == Scope.GLOBAL ? "global" : "file-local"; + String oldqual = prev.getScope() == Scope.GLOBAL ? "global" : "file-local"; + errorf(id, "conflicting %s declaration of '%s'", newqual, id.getName()); + if (prev.first != null) { + errorf(prev.first, "'%s' previously declared as %s here", id.getName(), oldqual); + } + } + + // Returns the union of accessible local and top-level symbols. + private Set getAllSymbols(Set predeclared) { + Set all = new HashSet<>(); + for (Block b = locals; b != null; b = b.parent) { + all.addAll(b.bindings.keySet()); + } + all.addAll(predeclared); + all.addAll(toplevel.keySet()); + return all; + } + + // Report an error if a load statement appears after another kind of statement. + private void checkLoadAfterStatement(List statements) { + Statement firstStatement = null; + + for (Statement statement : statements) { + // Ignore string literals (e.g. docstrings). + if (statement instanceof ExpressionStatement + && ((ExpressionStatement) statement).getExpression() instanceof StringLiteral) { + continue; + } + + if (statement instanceof LoadStatement) { + if (firstStatement == null) { + continue; + } + errorf(statement, "load statements must appear before any other statement"); + errorf(firstStatement, "\tfirst non-load statement appears here"); + } + + if (firstStatement == null) { + firstStatement = statement; + } + } + } + + /** + * Performs static checks, including resolution of identifiers in {@code file} in the environment + * defined by {@code module}. The StarlarkFile is mutated. Errors are appended to {@link + * StarlarkFile#errors}. + */ + public static void resolveFile(StarlarkFile file, Module module) { + Resolver r = new Resolver(file.errors, module, file.getOptions()); + ImmutableList stmts = file.getStatements(); + + // Check that load statements are on top. + if (r.options.requireLoadStatementsFirst()) { + r.checkLoadAfterStatement(stmts); + } + + ArrayList frame = new ArrayList<>(); + r.pushLocalBlock(file, frame, /*freevars=*/ null); + + // First pass: creating bindings for statements in this block. + r.createBindingsForBlock(stmts); + + // Second pass: visit all references. + r.visitAll(stmts); + + r.popLocalBlock(); + + // If the final statement is an expression, synthesize a return statement. + int n = stmts.size(); + if (n > 0 && stmts.get(n - 1) instanceof ExpressionStatement) { + Expression expr = ((ExpressionStatement) stmts.get(n - 1)).getExpression(); + stmts = + ImmutableList.builder() + .addAll(stmts.subList(0, n - 1)) + .add(ReturnStatement.make(expr)) + .build(); + } + + // Annotate with resolved information about the toplevel function. + file.setResolvedFunction( + new Function( + "", + file.getStartLocation(), + /*params=*/ ImmutableList.of(), + /*body=*/ stmts, + /*hasVarargs=*/ false, + /*hasKwargs=*/ false, + /*numKeywordOnlyParams=*/ 0, + frame, + /*freevars=*/ ImmutableList.of(), + r.globals)); + } + + /** + * Performs static checks, including resolution of identifiers in {@code expr} in the environment + * defined by {@code module}. This operation mutates the Expression. Syntax must be resolved + * before it is evaluated. + */ + public static Function resolveExpr(Expression expr, Module module, FileOptions options) + throws SyntaxError.Exception { + List errors = new ArrayList<>(); + Resolver r = new Resolver(errors, module, options); + + ArrayList frame = new ArrayList<>(); + r.pushLocalBlock(null, frame, /*freevars=*/ null); // for bindings in list comprehensions + r.visit(expr); + r.popLocalBlock(); + + if (!errors.isEmpty()) { + throw new SyntaxError.Exception(errors); + } + + // Return no-arg function that computes the expression. + return new Function( + "", + expr.getStartLocation(), + /*params=*/ ImmutableList.of(), + ImmutableList.of(ReturnStatement.make(expr)), + /*hasVarargs=*/ false, + /*hasKwargs=*/ false, + /*numKeywordOnlyParams=*/ 0, + frame, + /*freevars=*/ ImmutableList.of(), + r.globals); + } + + private void pushLocalBlock( + Node syntax, ArrayList frame, @Nullable ArrayList freevars) { + locals = new Block(locals, syntax, frame, freevars); + } + + private void popLocalBlock() { + locals = locals.parent; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/ReturnStatement.java b/third_party/bazel/main/java/net/starlark/java/syntax/ReturnStatement.java new file mode 100644 index 000000000..114941972 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/ReturnStatement.java @@ -0,0 +1,57 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import javax.annotation.Nullable; + +/** A syntax node for return statements. */ +public final class ReturnStatement extends Statement { + + private final int returnOffset; + @Nullable private final Expression result; + + ReturnStatement(FileLocations locs, int returnOffset, @Nullable Expression result) { + super(locs, Kind.RETURN); + this.returnOffset = returnOffset; + this.result = result; + } + + /** + * Returns a new return statement that returns expr. It is provided only for use by the evaluator, + * and will be removed when it switches to a compiled representation. + */ + static ReturnStatement make(Expression expr) { + return new ReturnStatement(expr.locs, expr.getStartOffset(), expr); + } + + @Nullable + public Expression getResult() { + return result; + } + + @Override + public int getStartOffset() { + return returnOffset; + } + + @Override + public int getEndOffset() { + return result != null ? result.getEndOffset() : returnOffset + "return".length(); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/SliceExpression.java b/third_party/bazel/main/java/net/starlark/java/syntax/SliceExpression.java new file mode 100644 index 000000000..a46ad8193 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/SliceExpression.java @@ -0,0 +1,82 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import javax.annotation.Nullable; + +/** Syntax node for a slice expression, {@code object[start:stop:step]}. */ +public final class SliceExpression extends Expression { + + private final Expression object; + private final int lbracketOffset; + @Nullable private final Expression start; + @Nullable private final Expression stop; + @Nullable private final Expression step; + private final int rbracketOffset; + + SliceExpression( + FileLocations locs, + Expression object, + int lbracketOffset, + Expression start, + Expression stop, + Expression step, + int rbracketOffset) { + super(locs, Kind.SLICE); + this.object = object; + this.lbracketOffset = lbracketOffset; + this.start = start; + this.stop = stop; + this.step = step; + this.rbracketOffset = rbracketOffset; + } + + public Expression getObject() { + return object; + } + + @Nullable + public Expression getStart() { + return start; + } + + @Nullable + public Expression getStop() { + return stop; + } + + @Nullable + public Expression getStep() { + return step; + } + + @Override + public int getStartOffset() { + return object.getStartOffset(); + } + + @Override + public int getEndOffset() { + return rbracketOffset + 1; + } + + public Location getLbracketLocation() { + return locs.getLocation(lbracketOffset); + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/StarlarkFile.java b/third_party/bazel/main/java/net/starlark/java/syntax/StarlarkFile.java new file mode 100644 index 000000000..8ebe6ea6b --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/StarlarkFile.java @@ -0,0 +1,168 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import com.google.common.collect.ImmutableList; +import java.util.Collections; +import java.util.List; +import javax.annotation.Nullable; + +/** + * Syntax tree for a Starlark file, such as a Bazel BUILD or .bzl file. + * + *

Call {@link #parse} to parse a file. Parser errors are recorded in the syntax tree (see {@link + * #errors}), which may be incomplete. + */ +public final class StarlarkFile extends Node { + + private final ImmutableList statements; + private final FileOptions options; + private final ImmutableList comments; + final List errors; // appended to by Resolver + + // set by resolver + @Nullable private Resolver.Function resolved; + + @Override + public int getStartOffset() { + return 0; + } + + @Override + public int getEndOffset() { + return locs.size(); + } + + private StarlarkFile( + FileLocations locs, + ImmutableList statements, + FileOptions options, + ImmutableList comments, + List errors) { + super(locs); + this.statements = statements; + this.options = options; + this.comments = comments; + this.errors = errors; + } + + /** + * Returns a new StarlarkFile whose statements are {@code getStatements().subList(start, end)}, + * and no comments. + * + * @deprecated This is a hack to support Bazel WORKSPACE files. + */ + @Deprecated + public StarlarkFile subTree(int start, int end) { + return new StarlarkFile( + this.locs, + this.statements.subList(start, end), + this.options, + /*comments=*/ ImmutableList.of(), + errors); + } + + /** + * Returns an unmodifiable view of the list of scanner, parser, and (perhaps) resolver errors + * accumulated in this Starlark file. + */ + public List errors() { + return Collections.unmodifiableList(errors); + } + + /** Returns errors().isEmpty(). */ + public boolean ok() { + return errors.isEmpty(); + } + + /** Returns an (immutable, ordered) list of statements in this BUILD file. */ + public ImmutableList getStatements() { + return statements; + } + + /** Returns an (immutable, ordered) list of comments in this BUILD file. */ + public ImmutableList getComments() { + return comments; + } + + @Override + public String toString() { + return ""; + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } + + void setResolvedFunction(Resolver.Function resolved) { + this.resolved = resolved; + } + + /** + * Returns information about the implicit function containing the top-level statements of the + * file. Set by the resolver. + */ + @Nullable + public Resolver.Function getResolvedFunction() { + return resolved; + } + + /** + * Parse a Starlark file. + * + *

A syntax tree is always returned, even in case of error. Errors are recorded in the tree. + * Example usage: + * + * 

+   * StarlarkFile file = StarlarkFile.parse(input, options);
+   * if (!file.ok()) {
+   *    Event.replayEventsOn(handler, file.errors());
+   *    ...
+   * }
+   *
+ */ + public static StarlarkFile parse(ParserInput input, FileOptions options) { + Parser.ParseResult result = Parser.parseFile(input, options); + return new StarlarkFile( + result.locs, result.statements, options, result.comments, result.errors); + } + + /** Parse a Starlark file with default options. */ + public static StarlarkFile parse(ParserInput input) { + return parse(input, FileOptions.DEFAULT); + } + + /** Returns the options specified when parsing this file. */ + public FileOptions getOptions() { + return options; + } + + /** Returns the name of this file, as specified to the parser. */ + public String getName() { + return locs.file(); + } + + /** A ParseProfiler records the start and end times of parse operations. */ + public interface ParseProfiler { + long start(); + + void end(long profileStartNanos, String filename); + } + + /** Installs a global hook that will be notified of parse operations. */ + public static void setParseProfiler(@Nullable ParseProfiler p) { + Parser.profiler = p; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Statement.java b/third_party/bazel/main/java/net/starlark/java/syntax/Statement.java new file mode 100644 index 000000000..717abe69d --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/Statement.java @@ -0,0 +1,50 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +/** Base class for all statements nodes in the AST. */ +public abstract class Statement extends Node { + + /** + * Kind of the statement. This is similar to using instanceof, except that it's more efficient and + * can be used in a switch/case. + */ + public enum Kind { + ASSIGNMENT, + EXPRESSION, + FLOW, + FOR, + DEF, + IF, + LOAD, + RETURN, + } + + // Materialize kind as a field so its accessor can be non-virtual. + private final Kind kind; + + Statement(FileLocations locs, Kind kind) { + super(locs); + this.kind = kind; + } + + /** + * Kind of the statement. This is similar to using instanceof, except that it's more efficient and + * can be used in a switch/case. + */ + // Final to avoid cost of virtual call (see #12967). + public final Kind kind() { + return kind; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/StringLiteral.java b/third_party/bazel/main/java/net/starlark/java/syntax/StringLiteral.java new file mode 100644 index 000000000..2fd8461bd --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/StringLiteral.java @@ -0,0 +1,98 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +import java.util.ArrayList; + +/** Syntax node for a string literal. */ +public final class StringLiteral extends Expression { + + // See skyframe.serialization.StringLiteralCodec for custom serialization logic. + + private final int startOffset; + private final String value; + private final int endOffset; + + StringLiteral(FileLocations locs, int startOffset, String value, int endOffset) { + super(locs, Kind.STRING_LITERAL); + this.startOffset = startOffset; + this.value = value; + this.endOffset = endOffset; + } + + /** Returns the value denoted by the string literal */ + public String getValue() { + return value; + } + + public Location getLocation() { + return locs.getLocation(startOffset); + } + + @Override + public int getStartOffset() { + return startOffset; + } + + @Override + public int getEndOffset() { + // TODO(adonovan): when we switch to compilation, + // making syntax trees ephemeral, we can afford to + // record the raw literal. This becomes: + // return startOffset + raw.length(). + return endOffset; + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } + + // -- hooks to support Skyframe serialization without creating a dependency -- + + /** Returns an opaque serializable object that may be passed to {@link #fromSerialization}. */ + public Object getFileLocations() { + return locs; + } + + /** + * Returns the value denoted by the Starlark string literal within s. + * + * @throws IllegalArgumentException if s does not contain a valid string literal. + */ + // TODO(bazel-team): We should in principle have an overload that allows non-default FileOptions. + // But currently no FileOptions affect the behavior of this method, except to possibly make it + // throw IAE on non-ASCII data. + public static String unquote(String s) { + // TODO(adonovan): once we have byte compilation, make this function + // independent of the Lexer, which should only validate string literals + // but not unquote them. Clients (e.g. the compiler) can unquote on demand. + ArrayList errors = new ArrayList<>(); + Lexer lexer = new Lexer(ParserInput.fromLines(s), errors, FileOptions.DEFAULT); + lexer.nextToken(); + if (!errors.isEmpty()) { + throw new IllegalArgumentException(errors.get(0).message()); + } + if (lexer.start != 0 || lexer.end != s.length() || lexer.kind != TokenKind.STRING) { + throw new IllegalArgumentException("invalid syntax"); + } + return (String) lexer.value; + } + + /** Constructs a StringLiteral from its serialized components. */ + public static StringLiteral fromSerialization( + Object fileLocations, int startOffset, String value, int endOffset) { + return new StringLiteral((FileLocations) fileLocations, startOffset, value, endOffset); + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/SyntaxError.java b/third_party/bazel/main/java/net/starlark/java/syntax/SyntaxError.java new file mode 100644 index 000000000..05487ad65 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/SyntaxError.java @@ -0,0 +1,99 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import java.util.List; + +/** + * A SyntaxError represents a static error associated with the syntax, such as a scanner or parse + * error, a structural problem, or a failure of identifier resolution. It records a description of + * the error and its location in the syntax. + */ +public final class SyntaxError { + + private final Location location; + private final String message; + + public SyntaxError(Location location, String message) { + this.location = Preconditions.checkNotNull(location); + this.message = Preconditions.checkNotNull(message); + } + + /** Returns the location of the error. */ + public Location location() { + return location; + } + + /** Returns a description of the error. */ + public String message() { + return message; + } + + /** Returns a string of the form {@code "foo.star:1:2: oops"}. */ + @Override + public String toString() { + return location + ": " + message; + } + + /** + * A SyntaxError.Exception is an exception holding one or more syntax errors. + * + *

SyntaxError.Exception is thrown by operations such as {@link Expression#parse}, which are + * "all or nothing". By contrast, {@link StarlarkFile#parse} does not throw an exception; instead, + * it records the accumulated scanner, parser, and optionally validation errors within the syntax + * tree, so that clients may obtain partial information from a damaged file. + * + *

Clients that fail abruptly when encountering parse errors are encouraged to throw + * SyntaxError.Exception, as in this example: + * + * 

+   * StarlarkFile file = StarlarkFile.parse(input);
+   * if (!file.ok()) {
+   *     throw new SyntaxError.Exception(file.errors());
+   * }
+   *
+ */ + @SuppressWarnings("JavaLangClash") + public static final class Exception extends java.lang.Exception { + + private final ImmutableList errors; + + /** Construct a SyntaxError from a non-empty list of errors. */ + public Exception(List errors) { + if (errors.isEmpty()) { + throw new IllegalArgumentException("no errors"); + } + this.errors = ImmutableList.copyOf(errors); + } + + /** Returns an immutable non-empty list of errors. */ + public ImmutableList errors() { + return errors; + } + + @Override + public String getMessage() { + String first = errors.get(0).message(); + if (errors.size() > 1) { + // TODO(adonovan): say ("+ n more errors") to avoid ambiguity. + return String.format("%s (+ %d more)", first, errors.size() - 1); + } else { + return first; + } + } + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/TokenKind.java b/third_party/bazel/main/java/net/starlark/java/syntax/TokenKind.java new file mode 100644 index 000000000..0b6ac6e69 --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/TokenKind.java @@ -0,0 +1,112 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net.starlark.java.syntax; + +/** A TokenKind represents the kind of a lexical token. */ +public enum TokenKind { + AMPERSAND("&"), + AMPERSAND_EQUALS("&="), + AND("and"), + AS("as"), + ASSERT("assert"), + BREAK("break"), + CARET("^"), + CARET_EQUALS("^="), + CLASS("class"), + COLON(":"), + COMMA(","), + CONTINUE("continue"), + DEF("def"), + DEL("del"), + DOT("."), + ELIF("elif"), + ELSE("else"), + EOF("EOF"), + EQUALS("="), + EQUALS_EQUALS("=="), + EXCEPT("except"), + FINALLY("finally"), + FLOAT("float literal"), + FOR("for"), + FROM("from"), + GLOBAL("global"), + GREATER(">"), + GREATER_EQUALS(">="), + GREATER_GREATER(">>"), + GREATER_GREATER_EQUALS(">>="), + IDENTIFIER("identifier"), + IF("if"), + ILLEGAL("illegal character"), + IMPORT("import"), + IN("in"), + INDENT("indent"), + INT("integer literal"), + IS("is"), + LAMBDA("lambda"), + LBRACE("{"), + LBRACKET("["), + LESS("<"), + LESS_EQUALS("<="), + LESS_LESS("<<"), + LESS_LESS_EQUALS("<<="), + LOAD("load"), + LPAREN("("), + MINUS("-"), + MINUS_EQUALS("-="), + NEWLINE("newline"), + NONLOCAL("nonlocal"), + NOT("not"), + NOT_EQUALS("!="), + NOT_IN("not in"), + OR("or"), + OUTDENT("outdent"), + PASS("pass"), + PERCENT("%"), + PERCENT_EQUALS("%="), + PIPE("|"), + PIPE_EQUALS("|="), + PLUS("+"), + PLUS_EQUALS("+="), + RAISE("raise"), + RBRACE("}"), + RBRACKET("]"), + RETURN("return"), + RPAREN(")"), + SEMI(";"), + SLASH("/"), + SLASH_EQUALS("/="), + SLASH_SLASH("//"), + SLASH_SLASH_EQUALS("//="), + STAR("*"), + STAR_EQUALS("*="), + STAR_STAR("**"), + STRING("string literal"), + TILDE("~"), + TRY("try"), + WHILE("while"), + WITH("with"), + YIELD("yield"); + + private final String name; + + private TokenKind(String name) { + this.name = name; + } + + @Override + public String toString() { + return name; + } +} diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/UnaryOperatorExpression.java b/third_party/bazel/main/java/net/starlark/java/syntax/UnaryOperatorExpression.java new file mode 100644 index 000000000..aa8e5104d --- /dev/null +++ b/third_party/bazel/main/java/net/starlark/java/syntax/UnaryOperatorExpression.java @@ -0,0 +1,63 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package net.starlark.java.syntax; + +/** A UnaryOperatorExpression represents a unary operator expression, 'op x'. */ +public final class UnaryOperatorExpression extends Expression { + + private final TokenKind op; // NOT, TILDE, MINUS or PLUS + private final int opOffset; + private final Expression x; + + UnaryOperatorExpression(FileLocations locs, TokenKind op, int opOffset, Expression x) { + super(locs, Kind.UNARY_OPERATOR); + this.op = op; + this.opOffset = opOffset; + this.x = x; + } + + /** Returns the operator. */ + public TokenKind getOperator() { + return op; + } + + @Override + public int getStartOffset() { + return opOffset; + } + + @Override + public int getEndOffset() { + return x.getEndOffset(); + } + + /** Returns the operand. */ + public Expression getX() { + return x; + } + + @Override + public String toString() { + // Note that this omits the parentheses for brevity, but is not correct in general due to + // operator precedence rules. For example, "(not False) in mylist" prints as + // "not False in mylist", which evaluates to opposite results in the case that mylist is empty. + // TODO(adonovan): record parentheses explicitly in syntax tree. + return (op == TokenKind.NOT ? "not " : op.toString()) + x; + } + + @Override + public void accept(NodeVisitor visitor) { + visitor.visit(this); + } +} From aba85312f41740a8a02eaa8e20b6676462af9d03 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Mon, 25 Nov 2024 18:15:56 +0000 Subject: [PATCH 2/7] Copy bazel/third_party/BUILD This is likely easier to keep in sync than reconciling with copybara/third_party/BUILD --- third_party/bazel/third_party/BUILD | 725 ++++++++++++++++++++++++++++ 1 file changed, 725 insertions(+) create mode 100644 third_party/bazel/third_party/BUILD diff --git a/third_party/bazel/third_party/BUILD b/third_party/bazel/third_party/BUILD new file mode 100644 index 000000000..e3c1e5da9 --- /dev/null +++ b/third_party/bazel/third_party/BUILD @@ -0,0 +1,725 @@ +load("@rules_java//java:defs.bzl", "java_import", "java_library", "java_plugin") +load("@rules_license//rules:license.bzl", "license") +load("//tools/distributions:distribution_rules.bzl", "distrib_jar_filegroup", "distrib_java_import") +load(":compiler_config_setting.bzl", "create_compiler_config_setting") + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "srcs", + srcs = glob(["**"]) + [ + "//third_party/allocation_instrumenter:srcs", + "//third_party/android_dex:srcs", + "//third_party/def_parser:srcs", + "//third_party/googleapis:srcs", + "//third_party/grpc:srcs", + "//third_party/grpc-java:srcs", + "//third_party/ijar:srcs", + "//third_party/jarjar:srcs", + "//third_party/java/android_databinding:srcs", + "//third_party/java/aosp_gradle_core:srcs", + "//third_party/java/j2objc:srcs", + "//third_party/java/j2objc-annotations:srcs", + "//third_party/java/jacoco:srcs", + "//third_party/java/javapoet:srcs", + "//third_party/java/jcommander:srcs", + "//third_party/java/proguard:srcs", + "//third_party/pprof:srcs", + "//third_party/protobuf:srcs", + "//third_party/py/abseil:srcs", + "//third_party/py/concurrent:srcs", + "//third_party/py/dataclasses:srcs", + "//third_party/py/frozendict:srcs", + "//third_party/py/mock:srcs", + "//third_party/remoteapis:srcs", + "//third_party/zlib:srcs", + ], +) + +# Filegroup to ship the sources to the Bazel embededded tools +# This filegroup should contains all GPL with classpath exception +# and LGPL code that we use in Bazel. +filegroup( + name = "gpl-srcs", + srcs = [], +) + +alias( + name = "apache_commons_collections", + actual = "@maven//:commons_collections_commons_collections", +) + +alias( + name = "apache_commons_lang", + actual = "@maven//:commons_lang_commons_lang", +) + +alias( + name = "apache_commons_compress", + actual = "@maven//:org_apache_commons_commons_compress", +) + +alias( + name = "apache_commons_pool2", + actual = "@maven//:org_apache_commons_commons_pool2", +) + +alias( + name = "apache_velocity", + actual = "@maven//:org_apache_velocity_velocity", +) + +java_library( + name = "api_client", + exports = [ + "@maven//:com_google_api_client_google_api_client", + "@maven//:com_google_api_client_google_api_client_gson", + "@maven//:com_google_http_client_google_http_client", + "@maven//:com_google_http_client_google_http_client_gson", + ], + runtime_deps = [ + ":gson", + ], +) + +distrib_java_import( + name = "asm", + enable_distributions = ["debian"], + jars = ["asm/asm-9.6.jar"], + srcjar = "asm/asm-9.6-sources.jar", +) + +java_import( + name = "asm-analysis", + jars = ["asm/asm-analysis-9.6.jar"], + srcjar = "asm/asm-analysis-9.6-sources.jar", + runtime_deps = [":asm-tree"], +) + +java_import( + name = "asm-commons", + jars = ["asm/asm-commons-9.6.jar"], + srcjar = "asm/asm-commons-9.6-sources.jar", + runtime_deps = [":asm-tree"], +) + +java_import( + name = "asm-tree", + jars = ["asm/asm-tree-9.6.jar"], + srcjar = "asm/asm-tree-9.6-sources.jar", + runtime_deps = [":asm"], +) + +java_import( + name = "asm-util", + jars = ["asm/asm-util-9.6.jar"], + srcjar = "asm/asm-util-9.6-sources.jar", + runtime_deps = [":asm-tree"], +) + +java_library( + name = "auth", + exports = [ + "@maven//:com_google_auth_google_auth_library_credentials", + "@maven//:com_google_auth_google_auth_library_oauth2_http", + ], + runtime_deps = [ + ":api_client", + ":guava", + ], +) + +java_plugin( + name = "auto_annotation_plugin", + processor_class = "com.google.auto.value.processor.AutoAnnotationProcessor", + deps = [ + ":apache_commons_collections", + ":apache_velocity", + ":auto_common", + ":auto_service_lib", + ":auto_value_lib", + ":guava", + ":jsr305", + ":tomcat_annotations_api", + ], +) + +alias( + name = "auto_common", + actual = "@maven//:com_google_auto_auto_common", +) + +java_library( + name = "auto_service", + exported_plugins = [ + ":auto_service_plugin", + ], + exports = [ + ":auto_service_api", + ], +) + +java_plugin( + name = "auto_service_plugin", + processor_class = "com.google.auto.service.processor.AutoServiceProcessor", + deps = [ + ":auto_common", + ":auto_service_lib", + ":guava", + ], +) + +java_library( + name = "auto_service_api", + exports = [ + "@maven//:com_google_auto_service_auto_service_annotations", + ], +) + +java_library( + name = "auto_service_lib", + exports = [ + "@maven//:com_google_auto_service_auto_service", + "@maven//:com_google_auto_service_auto_service_annotations", + ], +) + +java_plugin( + name = "auto_value_plugin", + processor_class = "com.google.auto.value.processor.AutoValueProcessor", + deps = [ + ":apache_commons_collections", + ":apache_velocity", + ":auto_common", + ":auto_service_lib", + ":auto_value_lib", + ":guava", + ":tomcat_annotations_api", + ], +) + +java_plugin( + name = "auto_oneof_plugin", + processor_class = "com.google.auto.value.processor.AutoOneOfProcessor", + deps = [ + ":apache_commons_collections", + ":apache_velocity", + ":auto_common", + ":auto_service_lib", + ":auto_value_lib", + ":guava", + ":tomcat_annotations_api", + ], +) + +java_plugin( + name = "auto_builder_plugin", + processor_class = "com.google.auto.value.processor.AutoBuilderProcessor", + deps = [ + ":apache_commons_collections", + ":apache_velocity", + ":auto_common", + ":auto_service_lib", + ":auto_value_lib", + ":guava", + ":tomcat_annotations_api", + ], +) + +java_plugin( + name = "auto_value_gson_plugin", + processor_class = "com.ryanharter.auto.value.gson.factory.AutoValueGsonAdapterFactoryProcessor", + deps = [ + "@maven//:com_ryanharter_auto_value_auto_value_gson_extension", + "@maven//:com_ryanharter_auto_value_auto_value_gson_factory", + ], +) + +java_library( + name = "auto_value", + exported_plugins = [ + ":auto_annotation_plugin", + ":auto_builder_plugin", + ":auto_oneof_plugin", + ":auto_value_plugin", + ":auto_value_gson_plugin", + ], + exports = [ + ":auto_value_api", + ":tomcat_annotations_api", + "@maven//:com_ryanharter_auto_value_auto_value_gson_runtime", + ], +) + +java_library( + name = "auto_value_api", + exports = [ + "@maven//:com_google_auto_value_auto_value_annotations", + ], +) + +java_library( + name = "auto_value_lib", + exports = [ + "@maven//:com_google_auto_value_auto_value", + "@maven//:com_google_auto_value_auto_value_annotations", + ], +) + +# For bootstrapping JavaBuilder +distrib_jar_filegroup( + name = "auto_value-jars", + srcs = [ + "@maven//:com_google_auto_value_auto_value_annotations_file", + "@maven//:com_google_auto_value_auto_value_file", + ], + enable_distributions = ["debian"], +) + +alias( + name = "checker_framework_annotations", + actual = "@maven//:org_checkerframework_checker_qual", +) + +alias( + name = "gson", + actual = "@maven//:com_google_code_gson_gson", +) + +alias( + name = "caffeine", + actual = "@maven//:com_github_ben_manes_caffeine_caffeine", +) + +# When using new classes from this dependency, make sure to update fastutil.proguard. +java_import( + name = "fastutil", + jars = [":fastutil_stripped_jar"], +) + +genrule( + name = "fastutil_stripped_jar", + srcs = [ + "@maven//:it_unimi_dsi_fastutil_file", + "@rules_java//toolchains:platformclasspath", + ], + outs = ["fastutil-stripped.jar"], + cmd = """ + $(location :proguard) \ + -injars $(execpath @maven//:it_unimi_dsi_fastutil_file) \ + -outjars $@ \ + -libraryjars $(execpath @rules_java//toolchains:platformclasspath) \ + @$(location //tools:fastutil.proguard) \ + | tail -n +2 # Skip the "ProGuard, version X" line + # Null out the file times stored in the jar to make the output reproducible. + TMPDIR=$$(mktemp -d) + trap 'rm -rf $$TMPDIR' EXIT + unzip -q $@ -d $$TMPDIR + rm $@ + find $$TMPDIR -type f -print0 | xargs -0 touch -t 198001010000.00 + OUTPUT="$$(pwd)/$@" + (cd $$TMPDIR && find . -type f | LC_ALL=C sort | zip -qDX0r@ "$$OUTPUT") + """, + tools = [ + ":proguard", + "//tools:fastutil.proguard", + ], +) + +java_binary( + name = "proguard", + main_class = "proguard.ProGuard", + runtime_deps = ["@maven//:com_guardsquare_proguard_base"], +) + +java_library( + name = "error_prone_annotations", + exports = [ + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_errorprone_error_prone_type_annotations", + ], +) + +distrib_jar_filegroup( + name = "error_prone_annotations-jar", + srcs = [ + "@maven//:com_google_errorprone_error_prone_annotations_file", + "@maven//:org_threeten_threeten_extra_file", + ], + enable_distributions = ["debian"], +) + +java_library( + name = "error_prone", + exports = [ + ":error_prone_annotations", + "@maven//:com_google_errorprone_error_prone_check_api", + "@maven//:com_google_errorprone_error_prone_core", + ], +) + +alias( + name = "jcip_annotations", + actual = "@maven//:com_github_stephenc_jcip_jcip_annotations", +) + +# For bootstrapping JavaBuilder +distrib_jar_filegroup( + name = "jcip_annotations-jars", + srcs = [ + "@maven//:com_github_stephenc_jcip_jcip_annotations_file", + ], + enable_distributions = ["debian"], +) + +alias( + name = "pcollections", + actual = "@maven//:org_pcollections_pcollections", +) + +# For bootstrapping JavaBuilder +filegroup( + name = "bootstrap_guava_and_error_prone-jars", + srcs = [ + ":error_prone_annotations-jar", + ":guava-jars", + ":jcip_annotations-jars", + ":jsr305-jars", + ], +) + +java_library( + name = "guava", + applicable_licenses = [":guava_license"], + exports = [ + ":error_prone_annotations", + ":jcip_annotations", + ":jsr305", + "@maven//:com_google_guava_guava", + ], +) + +license( + name = "guava_license", + package_name = "guava/31.1", + license_kinds = [ + "@rules_license//licenses/spdx:Apache-2.0", + ], + license_text = "guava/LICENSE", +) + +java_library( + name = "flogger", + applicable_licenses = [":flogger_license"], + exports = [ + "@maven//:com_google_flogger_flogger", + "@maven//:com_google_flogger_flogger_system_backend", + "@maven//:com_google_flogger_google_extensions", + ], +) + +license( + name = "flogger_license", + package_name = "flogger/0.5.1", + license_kinds = [ + "@rules_license//licenses/spdx:Apache-2.0", + ], +) + +distrib_jar_filegroup( + name = "flogger-jars", + srcs = [ + "@maven//:com_google_flogger_flogger_file", + "@maven//:com_google_flogger_flogger_system_backend_file", + "@maven//:com_google_flogger_google_extensions_file", + ], + enable_distributions = ["debian"], +) + +# For bootstrapping JavaBuilder +distrib_jar_filegroup( + name = "guava-jars", + srcs = ["@maven//:com_google_guava_guava_file"], + enable_distributions = ["debian"], +) + +# For desugaring the Guava jar. +distrib_jar_filegroup( + name = "guava-failureaccess-jar", + srcs = ["@maven//:com_google_guava_failureaccess_file"], + enable_distributions = ["debian"], +) + +alias( + name = "javax_activation", + actual = "@maven//:javax_activation_javax_activation_api", +) + +# javax.annotation.Generated is not included in the default root modules in 9, +# see: http://openjdk.java.net/jeps/320. +java_library( + name = "javax_annotations", + neverlink = 1, # @Generated is source-retention + exports = ["@maven//:javax_annotation_javax_annotation_api"], +) + +alias( + name = "rxjava3", + actual = "@maven//:io_reactivex_rxjava3_rxjava", +) + +alias( + name = "jsr305", + actual = "@maven//:com_google_code_findbugs_jsr305", +) + +# For bootstrapping JavaBuilder +distrib_jar_filegroup( + name = "jsr305-jars", + srcs = ["@maven//:com_google_code_findbugs_jsr305_file"], + enable_distributions = ["debian"], +) + +alias( + name = "jsr330_inject", + actual = "@maven//:javax_inject_javax_inject", +) + +UNNECESSARY_DYNAMIC_LIBRARIES = select({ + "//src/conditions:windows": "*.so *.jnilib", + "//src/conditions:darwin": "*.so *.dll", + "//src/conditions:linux_x86_64": "*.jnilib *.dll", + "//src/conditions:linux_s390x": "*.jnilib *.dll", + # The .so file is an x86/s390x one, so we can just remove it if the CPU is not x86/s390x + "//src/conditions:arm": "*.so *.jnilib *.dll", + "//src/conditions:linux_aarch64": "*.so *.jnilib *.dll", + "//src/conditions:linux_ppc": "*.so *.jnilib *.dll", + "//src/conditions:freebsd": "*.so *.jnilib *.dll", + "//src/conditions:openbsd": "*.so *.jnilib *.dll", + # Default is to play it safe -- better have a big binary than a slow binary + # The empty string means nothing is to be removed from the library; + # the rule command tests for the empty string explictly to avoid + # zip erroring when it finds nothing to remove. + "//conditions:default": "", +}) + +# Remove native libraries that are for a platform different from the one we are +# building Bazel for. +genrule( + name = "filter_netty_dynamic_libs", + srcs = select({ + "//src/conditions:darwin_arm64": ["@maven//:io_netty_netty_tcnative_boringssl_static_osx_aarch_64_file"], + "//src/conditions:darwin_x86_64": ["@maven//:io_netty_netty_tcnative_boringssl_static_osx_x86_64_file"], + "//src/conditions:linux_aarch64": ["@maven//:io_netty_netty_tcnative_boringssl_static_linux_aarch_64_file"], + "//src/conditions:linux_x86_64": ["@maven//:io_netty_netty_tcnative_boringssl_static_linux_x86_64_file"], + "//src/conditions:windows": ["@maven//:io_netty_netty_tcnative_boringssl_static_windows_x86_64_file"], + "//conditions:default": [], + }), + outs = ["netty_tcnative/netty-tcnative-filtered.jar"], + cmd = "cp $< $@ && " + + # Make sure we can write the output file, even if the input isn't writable. + "chmod +w $@ && " + + "zip -qd $@ */license/* " + UNNECESSARY_DYNAMIC_LIBRARIES, +) + +distrib_java_import( + name = "netty", + enable_distributions = ["debian"], + jars = [ + "@maven//:io_netty_netty_buffer_file", + "@maven//:io_netty_netty_codec_file", + "@maven//:io_netty_netty_codec_http2_file", + "@maven//:io_netty_netty_codec_http_file", + "@maven//:io_netty_netty_common_file", + "@maven//:io_netty_netty_handler_file", + "@maven//:io_netty_netty_handler_proxy_file", + "@maven//:io_netty_netty_resolver_file", + "@maven//:io_netty_netty_resolver_dns_file", + "@maven//:io_netty_netty_transport_file", + "@maven//:io_netty_netty_transport_classes_epoll_file", + "@maven//:io_netty_netty_transport_classes_kqueue_file", + ] + select({ + "//src/conditions:darwin_arm64": ["@maven//:io_netty_netty_transport_native_unix_common_osx_aarch_64_file"], + "//src/conditions:darwin_x86_64": ["@maven//:io_netty_netty_transport_native_unix_common_osx_x86_64_file"], + "//src/conditions:linux_aarch64": ["@maven//:io_netty_netty_transport_native_unix_common_linux_aarch_64_file"], + "//src/conditions:linux_x86_64": ["@maven//:io_netty_netty_transport_native_unix_common_linux_x86_64_file"], + "//conditions:default": ["@maven//:io_netty_netty_transport_native_unix_common_file"], + }) + select({ + "//src/conditions:darwin_arm64": ["@maven//:io_netty_netty_transport_native_kqueue_osx_aarch_64_file"], + "//src/conditions:darwin_x86_64": ["@maven//:io_netty_netty_transport_native_kqueue_osx_x86_64_file"], + "//conditions:default": [], + }) + select({ + "//src/conditions:linux_aarch64": ["@maven//:io_netty_netty_transport_native_epoll_linux_aarch_64_file"], + "//src/conditions:linux_x86_64": ["@maven//:io_netty_netty_transport_native_epoll_linux_x86_64_file"], + "//conditions:default": [], + }), +) + +distrib_java_import( + name = "netty_tcnative", + enable_distributions = ["debian"], + jars = [ + "@maven//:io_netty_netty_tcnative_classes_file", + ] + select({ + "//src/conditions:darwin_arm64": [":netty_tcnative/netty-tcnative-filtered.jar"], + "//src/conditions:darwin_x86_64": [":netty_tcnative/netty-tcnative-filtered.jar"], + "//src/conditions:linux_aarch64": [":netty_tcnative/netty-tcnative-filtered.jar"], + "//src/conditions:linux_x86_64": [":netty_tcnative/netty-tcnative-filtered.jar"], + "//src/conditions:windows": [":netty_tcnative/netty-tcnative-filtered.jar"], + "//conditions:default": [], + }), +) + +alias( + name = "tomcat_annotations_api", + actual = "@maven//:org_apache_tomcat_tomcat_annotations_api", +) + +# For bootstrapping JavaBuilder +distrib_jar_filegroup( + name = "tomcat_annotations_api-jars", + srcs = ["@maven//:org_apache_tomcat_tomcat_annotations_api_file"], + enable_distributions = ["debian"], +) + +alias( + name = "java-diff-utils", + actual = "@maven//:io_github_java_diff_utils_java_diff_utils", +) + +# Testing + +alias( + name = "compile_testing", + testonly = 1, + actual = "@maven//:com_google_testing_compile_compile_testing", +) + +alias( + name = "guava-testlib", + testonly = 1, + actual = "@maven//:com_google_guava_guava_testlib", +) + +# Not test_only due to //src/java_tools/junitrunner/java/com/google/testing/junit/junit4:runner +java_library( + name = "junit4", + exports = [ + "@maven//:junit_junit", + "@maven//:org_hamcrest_hamcrest_core", + ], +) + +alias( + name = "jimfs", + testonly = 1, + actual = "@maven//:com_google_jimfs_jimfs", +) + +alias( + name = "mockito", + testonly = 1, + actual = "@maven//:org_mockito_mockito_core", +) + +alias( + name = "turbine_direct", + actual = "@maven//:com_google_turbine_turbine", +) + +alias( + name = "turbine", + actual = "@maven//:com_google_turbine_turbine", +) + +java_library( + name = "truth", + testonly = 1, + exports = [ + "@maven//:com_google_truth_extensions_truth_java8_extension", + "@maven//:com_google_truth_extensions_truth_proto_extension", + "@maven//:com_google_truth_truth", + ], +) + +alias( + name = "xz", + actual = "@maven//:org_tukaani_xz", +) + +# To be used by the starlark example. +filegroup( + name = "junit4-jars", + srcs = [ + "@maven//:junit_junit_file", + "@maven//:org_hamcrest_hamcrest_core_file", + ], +) + +create_compiler_config_setting( + name = "windows_mingw", + value = "windows_mingw", +) + +create_compiler_config_setting( + name = "windows_msys64", + value = "windows_msys64", +) + +create_compiler_config_setting( + name = "windows_msys64_mingw64", + value = "windows_msys64_mingw64", +) + +create_compiler_config_setting( + name = "windows_clang", + value = "windows_clang", +) + +config_setting( + name = "k8", + values = {"host_cpu": "k8"}, +) + +config_setting( + name = "piii", + values = {"host_cpu": "piii"}, +) + +config_setting( + name = "arm", + values = {"host_cpu": "arm"}, +) + +config_setting( + name = "aarch64", + values = {"host_cpu": "aarch64"}, +) + +config_setting( + name = "freebsd", + values = {"host_cpu": "freebsd"}, +) + +config_setting( + name = "openbsd", + values = {"host_cpu": "openbsd"}, +) + +config_setting( + name = "s390x", + values = {"host_cpu": "s390x"}, +) + +config_setting( + name = "ppc", + values = {"host_cpu": "ppc"}, +) + +test_suite( + name = "all_windows_tests", + tests = [ + "//third_party/def_parser:windows_tests", + ], + visibility = ["//src:__pkg__"], +) From 0aebc7d91c1aeed722a26155311e6329f6ac0c43 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Mon, 25 Nov 2024 18:26:25 +0000 Subject: [PATCH 3/7] Remove unused targets, fix maven labels --- third_party/bazel/third_party/BUILD | 377 +++++----------------------- 1 file changed, 56 insertions(+), 321 deletions(-) diff --git a/third_party/bazel/third_party/BUILD b/third_party/bazel/third_party/BUILD index e3c1e5da9..ec291ee1e 100644 --- a/third_party/bazel/third_party/BUILD +++ b/third_party/bazel/third_party/BUILD @@ -1,41 +1,8 @@ load("@rules_java//java:defs.bzl", "java_import", "java_library", "java_plugin") load("@rules_license//rules:license.bzl", "license") -load("//tools/distributions:distribution_rules.bzl", "distrib_jar_filegroup", "distrib_java_import") -load(":compiler_config_setting.bzl", "create_compiler_config_setting") package(default_visibility = ["//visibility:public"]) -filegroup( - name = "srcs", - srcs = glob(["**"]) + [ - "//third_party/allocation_instrumenter:srcs", - "//third_party/android_dex:srcs", - "//third_party/def_parser:srcs", - "//third_party/googleapis:srcs", - "//third_party/grpc:srcs", - "//third_party/grpc-java:srcs", - "//third_party/ijar:srcs", - "//third_party/jarjar:srcs", - "//third_party/java/android_databinding:srcs", - "//third_party/java/aosp_gradle_core:srcs", - "//third_party/java/j2objc:srcs", - "//third_party/java/j2objc-annotations:srcs", - "//third_party/java/jacoco:srcs", - "//third_party/java/javapoet:srcs", - "//third_party/java/jcommander:srcs", - "//third_party/java/proguard:srcs", - "//third_party/pprof:srcs", - "//third_party/protobuf:srcs", - "//third_party/py/abseil:srcs", - "//third_party/py/concurrent:srcs", - "//third_party/py/dataclasses:srcs", - "//third_party/py/frozendict:srcs", - "//third_party/py/mock:srcs", - "//third_party/remoteapis:srcs", - "//third_party/zlib:srcs", - ], -) - # Filegroup to ship the sources to the Bazel embededded tools # This filegroup should contains all GPL with classpath exception # and LGPL code that we use in Bazel. @@ -46,49 +13,42 @@ filegroup( alias( name = "apache_commons_collections", - actual = "@maven//:commons_collections_commons_collections", + actual = "@copybara_maven//:commons_collections_commons_collections", ) alias( name = "apache_commons_lang", - actual = "@maven//:commons_lang_commons_lang", + actual = "@copybara_maven//:commons_lang_commons_lang", ) alias( name = "apache_commons_compress", - actual = "@maven//:org_apache_commons_commons_compress", + actual = "@copybara_maven//:org_apache_commons_commons_compress", ) alias( name = "apache_commons_pool2", - actual = "@maven//:org_apache_commons_commons_pool2", + actual = "@copybara_maven//:org_apache_commons_commons_pool2", ) alias( name = "apache_velocity", - actual = "@maven//:org_apache_velocity_velocity", + actual = "@copybara_maven//:org_apache_velocity_velocity", ) java_library( name = "api_client", exports = [ - "@maven//:com_google_api_client_google_api_client", - "@maven//:com_google_api_client_google_api_client_gson", - "@maven//:com_google_http_client_google_http_client", - "@maven//:com_google_http_client_google_http_client_gson", + "@copybara_maven//:com_google_api_client_google_api_client", + "@copybara_maven//:com_google_api_client_google_api_client_gson", + "@copybara_maven//:com_google_http_client_google_http_client", + "@copybara_maven//:com_google_http_client_google_http_client_gson", ], runtime_deps = [ ":gson", ], ) -distrib_java_import( - name = "asm", - enable_distributions = ["debian"], - jars = ["asm/asm-9.6.jar"], - srcjar = "asm/asm-9.6-sources.jar", -) - java_import( name = "asm-analysis", jars = ["asm/asm-analysis-9.6.jar"], @@ -120,8 +80,8 @@ java_import( java_library( name = "auth", exports = [ - "@maven//:com_google_auth_google_auth_library_credentials", - "@maven//:com_google_auth_google_auth_library_oauth2_http", + "@copybara_maven//:com_google_auth_google_auth_library_credentials", + "@copybara_maven//:com_google_auth_google_auth_library_oauth2_http", ], runtime_deps = [ ":api_client", @@ -146,7 +106,7 @@ java_plugin( alias( name = "auto_common", - actual = "@maven//:com_google_auto_auto_common", + actual = "@copybara_maven//:com_google_auto_auto_common", ) java_library( @@ -172,15 +132,15 @@ java_plugin( java_library( name = "auto_service_api", exports = [ - "@maven//:com_google_auto_service_auto_service_annotations", + "@copybara_maven//:com_google_auto_service_auto_service_annotations", ], ) java_library( name = "auto_service_lib", exports = [ - "@maven//:com_google_auto_service_auto_service", - "@maven//:com_google_auto_service_auto_service_annotations", + "@copybara_maven//:com_google_auto_service_auto_service", + "@copybara_maven//:com_google_auto_service_auto_service_annotations", ], ) @@ -230,8 +190,8 @@ java_plugin( name = "auto_value_gson_plugin", processor_class = "com.ryanharter.auto.value.gson.factory.AutoValueGsonAdapterFactoryProcessor", deps = [ - "@maven//:com_ryanharter_auto_value_auto_value_gson_extension", - "@maven//:com_ryanharter_auto_value_auto_value_gson_factory", + "@copybara_maven//:com_ryanharter_auto_value_auto_value_gson_extension", + "@copybara_maven//:com_ryanharter_auto_value_auto_value_gson_factory", ], ) @@ -247,145 +207,71 @@ java_library( exports = [ ":auto_value_api", ":tomcat_annotations_api", - "@maven//:com_ryanharter_auto_value_auto_value_gson_runtime", + "@copybara_maven//:com_ryanharter_auto_value_auto_value_gson_runtime", ], ) java_library( name = "auto_value_api", exports = [ - "@maven//:com_google_auto_value_auto_value_annotations", + "@copybara_maven//:com_google_auto_value_auto_value_annotations", ], ) java_library( name = "auto_value_lib", exports = [ - "@maven//:com_google_auto_value_auto_value", - "@maven//:com_google_auto_value_auto_value_annotations", - ], -) - -# For bootstrapping JavaBuilder -distrib_jar_filegroup( - name = "auto_value-jars", - srcs = [ - "@maven//:com_google_auto_value_auto_value_annotations_file", - "@maven//:com_google_auto_value_auto_value_file", + "@copybara_maven//:com_google_auto_value_auto_value", + "@copybara_maven//:com_google_auto_value_auto_value_annotations", ], - enable_distributions = ["debian"], ) alias( name = "checker_framework_annotations", - actual = "@maven//:org_checkerframework_checker_qual", + actual = "@copybara_maven//:org_checkerframework_checker_qual", ) alias( name = "gson", - actual = "@maven//:com_google_code_gson_gson", + actual = "@copybara_maven//:com_google_code_gson_gson", ) alias( name = "caffeine", - actual = "@maven//:com_github_ben_manes_caffeine_caffeine", -) - -# When using new classes from this dependency, make sure to update fastutil.proguard. -java_import( - name = "fastutil", - jars = [":fastutil_stripped_jar"], -) - -genrule( - name = "fastutil_stripped_jar", - srcs = [ - "@maven//:it_unimi_dsi_fastutil_file", - "@rules_java//toolchains:platformclasspath", - ], - outs = ["fastutil-stripped.jar"], - cmd = """ - $(location :proguard) \ - -injars $(execpath @maven//:it_unimi_dsi_fastutil_file) \ - -outjars $@ \ - -libraryjars $(execpath @rules_java//toolchains:platformclasspath) \ - @$(location //tools:fastutil.proguard) \ - | tail -n +2 # Skip the "ProGuard, version X" line - # Null out the file times stored in the jar to make the output reproducible. - TMPDIR=$$(mktemp -d) - trap 'rm -rf $$TMPDIR' EXIT - unzip -q $@ -d $$TMPDIR - rm $@ - find $$TMPDIR -type f -print0 | xargs -0 touch -t 198001010000.00 - OUTPUT="$$(pwd)/$@" - (cd $$TMPDIR && find . -type f | LC_ALL=C sort | zip -qDX0r@ "$$OUTPUT") - """, - tools = [ - ":proguard", - "//tools:fastutil.proguard", - ], + actual = "@copybara_maven//:com_github_ben_manes_caffeine_caffeine", ) java_binary( name = "proguard", main_class = "proguard.ProGuard", - runtime_deps = ["@maven//:com_guardsquare_proguard_base"], + runtime_deps = ["@copybara_maven//:com_guardsquare_proguard_base"], ) java_library( name = "error_prone_annotations", exports = [ - "@maven//:com_google_errorprone_error_prone_annotations", - "@maven//:com_google_errorprone_error_prone_type_annotations", + "@copybara_maven//:com_google_errorprone_error_prone_annotations", + "@copybara_maven//:com_google_errorprone_error_prone_type_annotations", ], ) -distrib_jar_filegroup( - name = "error_prone_annotations-jar", - srcs = [ - "@maven//:com_google_errorprone_error_prone_annotations_file", - "@maven//:org_threeten_threeten_extra_file", - ], - enable_distributions = ["debian"], -) - java_library( name = "error_prone", exports = [ ":error_prone_annotations", - "@maven//:com_google_errorprone_error_prone_check_api", - "@maven//:com_google_errorprone_error_prone_core", + "@copybara_maven//:com_google_errorprone_error_prone_check_api", + "@copybara_maven//:com_google_errorprone_error_prone_core", ], ) alias( name = "jcip_annotations", - actual = "@maven//:com_github_stephenc_jcip_jcip_annotations", -) - -# For bootstrapping JavaBuilder -distrib_jar_filegroup( - name = "jcip_annotations-jars", - srcs = [ - "@maven//:com_github_stephenc_jcip_jcip_annotations_file", - ], - enable_distributions = ["debian"], + actual = "@copybara_maven//:com_github_stephenc_jcip_jcip_annotations", ) alias( name = "pcollections", - actual = "@maven//:org_pcollections_pcollections", -) - -# For bootstrapping JavaBuilder -filegroup( - name = "bootstrap_guava_and_error_prone-jars", - srcs = [ - ":error_prone_annotations-jar", - ":guava-jars", - ":jcip_annotations-jars", - ":jsr305-jars", - ], + actual = "@copybara_maven//:org_pcollections_pcollections", ) java_library( @@ -395,7 +281,7 @@ java_library( ":error_prone_annotations", ":jcip_annotations", ":jsr305", - "@maven//:com_google_guava_guava", + "@copybara_maven//:com_google_guava_guava", ], ) @@ -412,9 +298,9 @@ java_library( name = "flogger", applicable_licenses = [":flogger_license"], exports = [ - "@maven//:com_google_flogger_flogger", - "@maven//:com_google_flogger_flogger_system_backend", - "@maven//:com_google_flogger_google_extensions", + "@copybara_maven//:com_google_flogger_flogger", + "@copybara_maven//:com_google_flogger_flogger_system_backend", + "@copybara_maven//:com_google_flogger_google_extensions", ], ) @@ -426,33 +312,9 @@ license( ], ) -distrib_jar_filegroup( - name = "flogger-jars", - srcs = [ - "@maven//:com_google_flogger_flogger_file", - "@maven//:com_google_flogger_flogger_system_backend_file", - "@maven//:com_google_flogger_google_extensions_file", - ], - enable_distributions = ["debian"], -) - -# For bootstrapping JavaBuilder -distrib_jar_filegroup( - name = "guava-jars", - srcs = ["@maven//:com_google_guava_guava_file"], - enable_distributions = ["debian"], -) - -# For desugaring the Guava jar. -distrib_jar_filegroup( - name = "guava-failureaccess-jar", - srcs = ["@maven//:com_google_guava_failureaccess_file"], - enable_distributions = ["debian"], -) - alias( name = "javax_activation", - actual = "@maven//:javax_activation_javax_activation_api", + actual = "@copybara_maven//:javax_activation_javax_activation_api", ) # javax.annotation.Generated is not included in the default root modules in 9, @@ -460,131 +322,32 @@ alias( java_library( name = "javax_annotations", neverlink = 1, # @Generated is source-retention - exports = ["@maven//:javax_annotation_javax_annotation_api"], + exports = ["@copybara_maven//:javax_annotation_javax_annotation_api"], ) alias( name = "rxjava3", - actual = "@maven//:io_reactivex_rxjava3_rxjava", + actual = "@copybara_maven//:io_reactivex_rxjava3_rxjava", ) alias( name = "jsr305", - actual = "@maven//:com_google_code_findbugs_jsr305", -) - -# For bootstrapping JavaBuilder -distrib_jar_filegroup( - name = "jsr305-jars", - srcs = ["@maven//:com_google_code_findbugs_jsr305_file"], - enable_distributions = ["debian"], + actual = "@copybara_maven//:com_google_code_findbugs_jsr305", ) alias( name = "jsr330_inject", - actual = "@maven//:javax_inject_javax_inject", -) - -UNNECESSARY_DYNAMIC_LIBRARIES = select({ - "//src/conditions:windows": "*.so *.jnilib", - "//src/conditions:darwin": "*.so *.dll", - "//src/conditions:linux_x86_64": "*.jnilib *.dll", - "//src/conditions:linux_s390x": "*.jnilib *.dll", - # The .so file is an x86/s390x one, so we can just remove it if the CPU is not x86/s390x - "//src/conditions:arm": "*.so *.jnilib *.dll", - "//src/conditions:linux_aarch64": "*.so *.jnilib *.dll", - "//src/conditions:linux_ppc": "*.so *.jnilib *.dll", - "//src/conditions:freebsd": "*.so *.jnilib *.dll", - "//src/conditions:openbsd": "*.so *.jnilib *.dll", - # Default is to play it safe -- better have a big binary than a slow binary - # The empty string means nothing is to be removed from the library; - # the rule command tests for the empty string explictly to avoid - # zip erroring when it finds nothing to remove. - "//conditions:default": "", -}) - -# Remove native libraries that are for a platform different from the one we are -# building Bazel for. -genrule( - name = "filter_netty_dynamic_libs", - srcs = select({ - "//src/conditions:darwin_arm64": ["@maven//:io_netty_netty_tcnative_boringssl_static_osx_aarch_64_file"], - "//src/conditions:darwin_x86_64": ["@maven//:io_netty_netty_tcnative_boringssl_static_osx_x86_64_file"], - "//src/conditions:linux_aarch64": ["@maven//:io_netty_netty_tcnative_boringssl_static_linux_aarch_64_file"], - "//src/conditions:linux_x86_64": ["@maven//:io_netty_netty_tcnative_boringssl_static_linux_x86_64_file"], - "//src/conditions:windows": ["@maven//:io_netty_netty_tcnative_boringssl_static_windows_x86_64_file"], - "//conditions:default": [], - }), - outs = ["netty_tcnative/netty-tcnative-filtered.jar"], - cmd = "cp $< $@ && " + - # Make sure we can write the output file, even if the input isn't writable. - "chmod +w $@ && " + - "zip -qd $@ */license/* " + UNNECESSARY_DYNAMIC_LIBRARIES, -) - -distrib_java_import( - name = "netty", - enable_distributions = ["debian"], - jars = [ - "@maven//:io_netty_netty_buffer_file", - "@maven//:io_netty_netty_codec_file", - "@maven//:io_netty_netty_codec_http2_file", - "@maven//:io_netty_netty_codec_http_file", - "@maven//:io_netty_netty_common_file", - "@maven//:io_netty_netty_handler_file", - "@maven//:io_netty_netty_handler_proxy_file", - "@maven//:io_netty_netty_resolver_file", - "@maven//:io_netty_netty_resolver_dns_file", - "@maven//:io_netty_netty_transport_file", - "@maven//:io_netty_netty_transport_classes_epoll_file", - "@maven//:io_netty_netty_transport_classes_kqueue_file", - ] + select({ - "//src/conditions:darwin_arm64": ["@maven//:io_netty_netty_transport_native_unix_common_osx_aarch_64_file"], - "//src/conditions:darwin_x86_64": ["@maven//:io_netty_netty_transport_native_unix_common_osx_x86_64_file"], - "//src/conditions:linux_aarch64": ["@maven//:io_netty_netty_transport_native_unix_common_linux_aarch_64_file"], - "//src/conditions:linux_x86_64": ["@maven//:io_netty_netty_transport_native_unix_common_linux_x86_64_file"], - "//conditions:default": ["@maven//:io_netty_netty_transport_native_unix_common_file"], - }) + select({ - "//src/conditions:darwin_arm64": ["@maven//:io_netty_netty_transport_native_kqueue_osx_aarch_64_file"], - "//src/conditions:darwin_x86_64": ["@maven//:io_netty_netty_transport_native_kqueue_osx_x86_64_file"], - "//conditions:default": [], - }) + select({ - "//src/conditions:linux_aarch64": ["@maven//:io_netty_netty_transport_native_epoll_linux_aarch_64_file"], - "//src/conditions:linux_x86_64": ["@maven//:io_netty_netty_transport_native_epoll_linux_x86_64_file"], - "//conditions:default": [], - }), -) - -distrib_java_import( - name = "netty_tcnative", - enable_distributions = ["debian"], - jars = [ - "@maven//:io_netty_netty_tcnative_classes_file", - ] + select({ - "//src/conditions:darwin_arm64": [":netty_tcnative/netty-tcnative-filtered.jar"], - "//src/conditions:darwin_x86_64": [":netty_tcnative/netty-tcnative-filtered.jar"], - "//src/conditions:linux_aarch64": [":netty_tcnative/netty-tcnative-filtered.jar"], - "//src/conditions:linux_x86_64": [":netty_tcnative/netty-tcnative-filtered.jar"], - "//src/conditions:windows": [":netty_tcnative/netty-tcnative-filtered.jar"], - "//conditions:default": [], - }), + actual = "@copybara_maven//:javax_inject_javax_inject", ) alias( name = "tomcat_annotations_api", - actual = "@maven//:org_apache_tomcat_tomcat_annotations_api", -) - -# For bootstrapping JavaBuilder -distrib_jar_filegroup( - name = "tomcat_annotations_api-jars", - srcs = ["@maven//:org_apache_tomcat_tomcat_annotations_api_file"], - enable_distributions = ["debian"], + actual = "@copybara_maven//:org_apache_tomcat_tomcat_annotations_api", ) alias( name = "java-diff-utils", - actual = "@maven//:io_github_java_diff_utils_java_diff_utils", + actual = "@copybara_maven//:io_github_java_diff_utils_java_diff_utils", ) # Testing @@ -592,90 +355,70 @@ alias( alias( name = "compile_testing", testonly = 1, - actual = "@maven//:com_google_testing_compile_compile_testing", + actual = "@copybara_maven//:com_google_testing_compile_compile_testing", ) alias( name = "guava-testlib", testonly = 1, - actual = "@maven//:com_google_guava_guava_testlib", + actual = "@copybara_maven//:com_google_guava_guava_testlib", ) # Not test_only due to //src/java_tools/junitrunner/java/com/google/testing/junit/junit4:runner java_library( name = "junit4", exports = [ - "@maven//:junit_junit", - "@maven//:org_hamcrest_hamcrest_core", + "@copybara_maven//:junit_junit", + "@copybara_maven//:org_hamcrest_hamcrest_core", ], ) alias( name = "jimfs", testonly = 1, - actual = "@maven//:com_google_jimfs_jimfs", + actual = "@copybara_maven//:com_google_jimfs_jimfs", ) alias( name = "mockito", testonly = 1, - actual = "@maven//:org_mockito_mockito_core", + actual = "@copybara_maven//:org_mockito_mockito_core", ) alias( name = "turbine_direct", - actual = "@maven//:com_google_turbine_turbine", + actual = "@copybara_maven//:com_google_turbine_turbine", ) alias( name = "turbine", - actual = "@maven//:com_google_turbine_turbine", + actual = "@copybara_maven//:com_google_turbine_turbine", ) java_library( name = "truth", testonly = 1, exports = [ - "@maven//:com_google_truth_extensions_truth_java8_extension", - "@maven//:com_google_truth_extensions_truth_proto_extension", - "@maven//:com_google_truth_truth", + "@copybara_maven//:com_google_truth_extensions_truth_java8_extension", + "@copybara_maven//:com_google_truth_extensions_truth_proto_extension", + "@copybara_maven//:com_google_truth_truth", ], ) alias( name = "xz", - actual = "@maven//:org_tukaani_xz", + actual = "@copybara_maven//:org_tukaani_xz", ) # To be used by the starlark example. filegroup( name = "junit4-jars", srcs = [ - "@maven//:junit_junit_file", - "@maven//:org_hamcrest_hamcrest_core_file", + "@copybara_maven//:junit_junit_file", + "@copybara_maven//:org_hamcrest_hamcrest_core_file", ], ) -create_compiler_config_setting( - name = "windows_mingw", - value = "windows_mingw", -) - -create_compiler_config_setting( - name = "windows_msys64", - value = "windows_msys64", -) - -create_compiler_config_setting( - name = "windows_msys64_mingw64", - value = "windows_msys64_mingw64", -) - -create_compiler_config_setting( - name = "windows_clang", - value = "windows_clang", -) - config_setting( name = "k8", values = {"host_cpu": "k8"}, @@ -715,11 +458,3 @@ config_setting( name = "ppc", values = {"host_cpu": "ppc"}, ) - -test_suite( - name = "all_windows_tests", - tests = [ - "//third_party/def_parser:windows_tests", - ], - visibility = ["//src:__pkg__"], -) From 5136f015571f6c88d54fe3602712131481db3315 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Mon, 25 Nov 2024 18:26:38 +0000 Subject: [PATCH 4/7] Update labels and visibility --- third_party/BUILD | 14 ++++----- third_party/bazel/BUILD | 12 ++++++++ .../bazel/main/java/net/starlark/java/BUILD | 2 +- .../main/java/net/starlark/java/annot/BUILD | 14 ++++----- .../net/starlark/java/annot/processor/BUILD | 16 +++++----- .../main/java/net/starlark/java/eval/BUILD | 30 +++++++++---------- .../java/net/starlark/java/lib/json/BUILD | 12 ++++---- .../java/net/starlark/java/spelling/BUILD | 10 +++---- .../main/java/net/starlark/java/syntax/BUILD | 14 ++++----- 9 files changed, 66 insertions(+), 58 deletions(-) diff --git a/third_party/BUILD b/third_party/BUILD index e481125c0..521457b56 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -166,10 +166,10 @@ java_library( java_library( name = "starlark", exports = [ - "@io_bazel//src/main/java/net/starlark/java/annot", - "@io_bazel//src/main/java/net/starlark/java/eval", - "@io_bazel//src/main/java/net/starlark/java/lib/json", - "@io_bazel//src/main/java/net/starlark/java/syntax", + "//third_party/bazel/main/java/net/starlark/java/annot", + "//third_party/bazel/main/java/net/starlark/java/eval", + "//third_party/bazel/main/java/net/starlark/java/lib/json", + "//third_party/bazel/main/java/net/starlark/java/syntax", ], ) @@ -186,7 +186,7 @@ java_library( java_library( name = "error_prone", exports = [ - "@io_bazel//third_party:error_prone_annotations", + "//third_party/bazel/third_party:error_prone_annotations", ], ) @@ -225,7 +225,3 @@ java_library( "@copybara_maven//:com_google_protobuf_protobuf_lite", ], ) - -# Required temporarily until @io_bazel//src/main/java/com/google/devtools/build/lib/syntax/... -# is fixed -exports_files(["bazel.patch"]) diff --git a/third_party/bazel/BUILD b/third_party/bazel/BUILD index 0a19ed091..cb1e28f9b 100644 --- a/third_party/bazel/BUILD +++ b/third_party/bazel/BUILD @@ -1,3 +1,5 @@ +load("@rules_license//rules:license.bzl", "license") + licenses(["notice"]) # Apache 2.0 package( @@ -6,6 +8,16 @@ package( ], ) +license( + name = "license", + package_name = "bazelbuild/bazel", + copyright_notice = "Copyright © 2014 The Bazel Authors. All rights reserved.", + license_kinds = [ + "@rules_license//licenses/spdx:Apache-2.0", + ], + license_text = "LICENSE", +) + java_library( name = "shell", srcs = glob(["main/**/*.java"]), diff --git a/third_party/bazel/main/java/net/starlark/java/BUILD b/third_party/bazel/main/java/net/starlark/java/BUILD index 06d180dc3..8b5b4dfa5 100644 --- a/third_party/bazel/main/java/net/starlark/java/BUILD +++ b/third_party/bazel/main/java/net/starlark/java/BUILD @@ -12,8 +12,8 @@ filegroup( package_group( name = "starlark", packages = [ - "//src/main/java/net/starlark/java/...", "//src/test/java/net/starlark/java/...", + "//third_party/bazel/main/java/net/starlark/java/...", ], ) diff --git a/third_party/bazel/main/java/net/starlark/java/annot/BUILD b/third_party/bazel/main/java/net/starlark/java/annot/BUILD index d9bed9bf5..6752e200c 100644 --- a/third_party/bazel/main/java/net/starlark/java/annot/BUILD +++ b/third_party/bazel/main/java/net/starlark/java/annot/BUILD @@ -1,8 +1,8 @@ load("@rules_java//java:defs.bzl", "java_library") package( - default_applicable_licenses = ["//:license"], - default_visibility = ["//src:__subpackages__"], + default_applicable_licenses = ["//third_party/bazel:license"], + default_visibility = ["//visibility:public"], ) filegroup( @@ -15,8 +15,8 @@ filegroup( # Depending on this library adds annotations processing to the build. java_library( name = "annot", - exported_plugins = ["//src/main/java/net/starlark/java/annot/processor"], - visibility = ["//src/main/java/net/starlark/java:clients"], + exported_plugins = ["//third_party/bazel/main/java/net/starlark/java/annot/processor"], + visibility = ["//third_party/bazel/main/java/net/starlark/java:clients"], exports = [":annot_sans_processor"], ) @@ -24,9 +24,9 @@ java_library( java_library( name = "annot_sans_processor", srcs = glob(["*.java"]), - visibility = ["//src/main/java/net/starlark/java/annot/processor:__pkg__"], + visibility = ["//third_party/bazel/main/java/net/starlark/java/annot/processor:__pkg__"], deps = [ - "//third_party:guava", - "//third_party:jsr305", + "//third_party/bazel/third_party:guava", + "//third_party/bazel/third_party:jsr305", ], ) diff --git a/third_party/bazel/main/java/net/starlark/java/annot/processor/BUILD b/third_party/bazel/main/java/net/starlark/java/annot/processor/BUILD index d9fd1925a..bef38a1ba 100644 --- a/third_party/bazel/main/java/net/starlark/java/annot/processor/BUILD +++ b/third_party/bazel/main/java/net/starlark/java/annot/processor/BUILD @@ -1,8 +1,8 @@ load("@rules_java//java:defs.bzl", "java_library", "java_plugin") package( - default_applicable_licenses = ["//:license"], - default_visibility = ["//src:__subpackages__"], + default_applicable_licenses = ["//third_party/bazel:license"], + default_visibility = ["//visibility:public"], ) filegroup( @@ -15,18 +15,18 @@ filegroup( java_plugin( name = "processor", processor_class = "net.starlark.java.annot.processor.StarlarkMethodProcessor", - visibility = ["//src/main/java/net/starlark/java:starlark"], + visibility = ["//third_party/bazel/main/java/net/starlark/java:starlark"], deps = [":processor_lib"], ) java_library( name = "processor_lib", srcs = ["StarlarkMethodProcessor.java"], - visibility = ["//src/main/java/net/starlark/java:starlark"], + visibility = ["//third_party/bazel/main/java/net/starlark/java:starlark"], deps = [ - "//src/main/java/net/starlark/java/annot:annot_sans_processor", - "//third_party:error_prone_annotations", - "//third_party:guava", - "//third_party:jsr305", + "//third_party/bazel/main/java/net/starlark/java/annot:annot_sans_processor", + "//third_party/bazel/third_party:error_prone_annotations", + "//third_party/bazel/third_party:guava", + "//third_party/bazel/third_party:jsr305", ], ) diff --git a/third_party/bazel/main/java/net/starlark/java/eval/BUILD b/third_party/bazel/main/java/net/starlark/java/eval/BUILD index 93f62b94c..5cc552274 100644 --- a/third_party/bazel/main/java/net/starlark/java/eval/BUILD +++ b/third_party/bazel/main/java/net/starlark/java/eval/BUILD @@ -7,7 +7,7 @@ licenses(["notice"]) # Apache 2.0 filegroup( name = "srcs", srcs = glob(["**"]), - visibility = ["//src/main/java/net/starlark/java:bazel"], + visibility = ["//third_party/bazel/main/java/net/starlark/java:bazel"], ) # The Starlark evaluator @@ -61,17 +61,17 @@ java_library( "SymbolGenerator.java", "Tuple.java", ], - visibility = ["//src/main/java/net/starlark/java:clients"], + visibility = ["//third_party/bazel/main/java/net/starlark/java:clients"], deps = [ # Do not add Bazel or Google dependencies here! - "//src/main/java/net/starlark/java/annot", - "//src/main/java/net/starlark/java/spelling", - "//src/main/java/net/starlark/java/syntax", - "//third_party:error_prone_annotations", - "//third_party:auto_value", - "//third_party:flogger", - "//third_party:guava", - "//third_party:jsr305", + "//third_party/bazel/main/java/net/starlark/java/annot", + "//third_party/bazel/main/java/net/starlark/java/spelling", + "//third_party/bazel/main/java/net/starlark/java/syntax", + "//third_party/bazel/third_party:error_prone_annotations", + "//third_party/bazel/third_party:auto_value", + "//third_party/bazel/third_party:flogger", + "//third_party/bazel/third_party:guava", + "//third_party/bazel/third_party:jsr305", ], ) @@ -82,11 +82,11 @@ java_library( filegroup( name = "cpu_profiler", srcs = select({ - "//src/conditions:darwin": [":libcpu_profiler.dylib"], - "//src/conditions:windows": [":cpu_profiler.dll"], + "@platforms//os:macos": [":libcpu_profiler.dylib"], + "@platforms//os:windows": [":cpu_profiler.dll"], "//conditions:default": [":libcpu_profiler.so"], # POSIX }), - visibility = ["//src/main/java/net/starlark/java:clients"], + visibility = ["//third_party/bazel/main/java/net/starlark/java:clients"], ) genrule( @@ -107,8 +107,8 @@ genrule( cc_binary( name = "libcpu_profiler.so", srcs = select({ - "//src/conditions:darwin": ["cpu_profiler_posix.cc"], - "//src/conditions:linux": ["cpu_profiler_posix.cc"], + "@platforms//os:macos": ["cpu_profiler_posix.cc"], + "@platforms//os:linux": ["cpu_profiler_posix.cc"], "//conditions:default": ["cpu_profiler_unimpl.cc"], }), linkshared = 1, diff --git a/third_party/bazel/main/java/net/starlark/java/lib/json/BUILD b/third_party/bazel/main/java/net/starlark/java/lib/json/BUILD index 0a6d435b4..16036cf5c 100644 --- a/third_party/bazel/main/java/net/starlark/java/lib/json/BUILD +++ b/third_party/bazel/main/java/net/starlark/java/lib/json/BUILD @@ -1,8 +1,8 @@ load("@rules_java//java:defs.bzl", "java_library") package( - default_applicable_licenses = ["//:license"], - default_visibility = ["//src:__subpackages__"], + default_applicable_licenses = ["//third_party/bazel:license"], + default_visibility = ["//visibility:public"], ) filegroup( @@ -15,10 +15,10 @@ filegroup( java_library( name = "json", srcs = ["Json.java"], - visibility = ["//src/main/java/net/starlark/java:clients"], + visibility = ["//third_party/bazel/main/java/net/starlark/java:clients"], deps = [ - "//src/main/java/net/starlark/java/annot", - "//src/main/java/net/starlark/java/eval", - "//src/main/java/net/starlark/java/syntax", + "//third_party/bazel/main/java/net/starlark/java/annot", + "//third_party/bazel/main/java/net/starlark/java/eval", + "//third_party/bazel/main/java/net/starlark/java/syntax", ], ) diff --git a/third_party/bazel/main/java/net/starlark/java/spelling/BUILD b/third_party/bazel/main/java/net/starlark/java/spelling/BUILD index 48164c993..9c72af1bc 100644 --- a/third_party/bazel/main/java/net/starlark/java/spelling/BUILD +++ b/third_party/bazel/main/java/net/starlark/java/spelling/BUILD @@ -3,8 +3,8 @@ load("@rules_java//java:defs.bzl", "java_library") package( - default_applicable_licenses = ["//:license"], - default_visibility = ["//src:__subpackages__"], + default_applicable_licenses = ["//third_party/bazel:license"], + default_visibility = ["//visibility:public"], ) filegroup( @@ -18,10 +18,10 @@ java_library( srcs = ["SpellChecker.java"], # The SpellChecker is available to Starlark and Bazel, but not the world. visibility = [ - "//src/main/java/net/starlark/java:bazel", - "//src/main/java/net/starlark/java:starlark", + "//third_party/bazel/main/java/net/starlark/java:bazel", + "//third_party/bazel/main/java/net/starlark/java:starlark", ], deps = [ - "//third_party:jsr305", + "//third_party/bazel/third_party:jsr305", ], ) diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/BUILD b/third_party/bazel/main/java/net/starlark/java/syntax/BUILD index 9b13f7867..2940e500e 100644 --- a/third_party/bazel/main/java/net/starlark/java/syntax/BUILD +++ b/third_party/bazel/main/java/net/starlark/java/syntax/BUILD @@ -3,8 +3,8 @@ load("@rules_java//java:defs.bzl", "java_library") package( - default_applicable_licenses = ["//:license"], - default_visibility = ["//src:__subpackages__"], + default_applicable_licenses = ["//third_party/bazel:license"], + default_visibility = ["//visibility:public"], ) filegroup( @@ -60,12 +60,12 @@ java_library( "TokenKind.java", "UnaryOperatorExpression.java", ], - visibility = ["//src/main/java/net/starlark/java:clients"], + visibility = ["//third_party/bazel/main/java/net/starlark/java:clients"], # Do not add Bazel or Google dependencies here! deps = [ - "//src/main/java/net/starlark/java/spelling", - "//third_party:auto_value", - "//third_party:guava", - "//third_party:jsr305", + "//third_party/bazel/main/java/net/starlark/java/spelling", + "//third_party/bazel/third_party:auto_value", + "//third_party/bazel/third_party:guava", + "//third_party/bazel/third_party:jsr305", ], ) From 1609c271775bf868fbc8796734d69494d05712c6 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Mon, 25 Nov 2024 18:33:53 +0000 Subject: [PATCH 5/7] Remove more unused third parties from bazel --- third_party/bazel/third_party/BUILD | 192 +--------------------------- 1 file changed, 1 insertion(+), 191 deletions(-) diff --git a/third_party/bazel/third_party/BUILD b/third_party/bazel/third_party/BUILD index ec291ee1e..622ba4f3d 100644 --- a/third_party/bazel/third_party/BUILD +++ b/third_party/bazel/third_party/BUILD @@ -1,4 +1,4 @@ -load("@rules_java//java:defs.bzl", "java_import", "java_library", "java_plugin") +load("@rules_java//java:defs.bzl", "java_library", "java_plugin") load("@rules_license//rules:license.bzl", "license") package(default_visibility = ["//visibility:public"]) @@ -16,79 +16,11 @@ alias( actual = "@copybara_maven//:commons_collections_commons_collections", ) -alias( - name = "apache_commons_lang", - actual = "@copybara_maven//:commons_lang_commons_lang", -) - -alias( - name = "apache_commons_compress", - actual = "@copybara_maven//:org_apache_commons_commons_compress", -) - -alias( - name = "apache_commons_pool2", - actual = "@copybara_maven//:org_apache_commons_commons_pool2", -) - alias( name = "apache_velocity", actual = "@copybara_maven//:org_apache_velocity_velocity", ) -java_library( - name = "api_client", - exports = [ - "@copybara_maven//:com_google_api_client_google_api_client", - "@copybara_maven//:com_google_api_client_google_api_client_gson", - "@copybara_maven//:com_google_http_client_google_http_client", - "@copybara_maven//:com_google_http_client_google_http_client_gson", - ], - runtime_deps = [ - ":gson", - ], -) - -java_import( - name = "asm-analysis", - jars = ["asm/asm-analysis-9.6.jar"], - srcjar = "asm/asm-analysis-9.6-sources.jar", - runtime_deps = [":asm-tree"], -) - -java_import( - name = "asm-commons", - jars = ["asm/asm-commons-9.6.jar"], - srcjar = "asm/asm-commons-9.6-sources.jar", - runtime_deps = [":asm-tree"], -) - -java_import( - name = "asm-tree", - jars = ["asm/asm-tree-9.6.jar"], - srcjar = "asm/asm-tree-9.6-sources.jar", - runtime_deps = [":asm"], -) - -java_import( - name = "asm-util", - jars = ["asm/asm-util-9.6.jar"], - srcjar = "asm/asm-util-9.6-sources.jar", - runtime_deps = [":asm-tree"], -) - -java_library( - name = "auth", - exports = [ - "@copybara_maven//:com_google_auth_google_auth_library_credentials", - "@copybara_maven//:com_google_auth_google_auth_library_oauth2_http", - ], - runtime_deps = [ - ":api_client", - ":guava", - ], -) - java_plugin( name = "auto_annotation_plugin", processor_class = "com.google.auto.value.processor.AutoAnnotationProcessor", @@ -236,17 +168,6 @@ alias( actual = "@copybara_maven//:com_google_code_gson_gson", ) -alias( - name = "caffeine", - actual = "@copybara_maven//:com_github_ben_manes_caffeine_caffeine", -) - -java_binary( - name = "proguard", - main_class = "proguard.ProGuard", - runtime_deps = ["@copybara_maven//:com_guardsquare_proguard_base"], -) - java_library( name = "error_prone_annotations", exports = [ @@ -255,25 +176,11 @@ java_library( ], ) -java_library( - name = "error_prone", - exports = [ - ":error_prone_annotations", - "@copybara_maven//:com_google_errorprone_error_prone_check_api", - "@copybara_maven//:com_google_errorprone_error_prone_core", - ], -) - alias( name = "jcip_annotations", actual = "@copybara_maven//:com_github_stephenc_jcip_jcip_annotations", ) -alias( - name = "pcollections", - actual = "@copybara_maven//:org_pcollections_pcollections", -) - java_library( name = "guava", applicable_licenses = [":guava_license"], @@ -312,113 +219,16 @@ license( ], ) -alias( - name = "javax_activation", - actual = "@copybara_maven//:javax_activation_javax_activation_api", -) - -# javax.annotation.Generated is not included in the default root modules in 9, -# see: http://openjdk.java.net/jeps/320. -java_library( - name = "javax_annotations", - neverlink = 1, # @Generated is source-retention - exports = ["@copybara_maven//:javax_annotation_javax_annotation_api"], -) - -alias( - name = "rxjava3", - actual = "@copybara_maven//:io_reactivex_rxjava3_rxjava", -) - alias( name = "jsr305", actual = "@copybara_maven//:com_google_code_findbugs_jsr305", ) -alias( - name = "jsr330_inject", - actual = "@copybara_maven//:javax_inject_javax_inject", -) - alias( name = "tomcat_annotations_api", actual = "@copybara_maven//:org_apache_tomcat_tomcat_annotations_api", ) -alias( - name = "java-diff-utils", - actual = "@copybara_maven//:io_github_java_diff_utils_java_diff_utils", -) - -# Testing - -alias( - name = "compile_testing", - testonly = 1, - actual = "@copybara_maven//:com_google_testing_compile_compile_testing", -) - -alias( - name = "guava-testlib", - testonly = 1, - actual = "@copybara_maven//:com_google_guava_guava_testlib", -) - -# Not test_only due to //src/java_tools/junitrunner/java/com/google/testing/junit/junit4:runner -java_library( - name = "junit4", - exports = [ - "@copybara_maven//:junit_junit", - "@copybara_maven//:org_hamcrest_hamcrest_core", - ], -) - -alias( - name = "jimfs", - testonly = 1, - actual = "@copybara_maven//:com_google_jimfs_jimfs", -) - -alias( - name = "mockito", - testonly = 1, - actual = "@copybara_maven//:org_mockito_mockito_core", -) - -alias( - name = "turbine_direct", - actual = "@copybara_maven//:com_google_turbine_turbine", -) - -alias( - name = "turbine", - actual = "@copybara_maven//:com_google_turbine_turbine", -) - -java_library( - name = "truth", - testonly = 1, - exports = [ - "@copybara_maven//:com_google_truth_extensions_truth_java8_extension", - "@copybara_maven//:com_google_truth_extensions_truth_proto_extension", - "@copybara_maven//:com_google_truth_truth", - ], -) - -alias( - name = "xz", - actual = "@copybara_maven//:org_tukaani_xz", -) - -# To be used by the starlark example. -filegroup( - name = "junit4-jars", - srcs = [ - "@copybara_maven//:junit_junit_file", - "@copybara_maven//:org_hamcrest_hamcrest_core_file", - ], -) - config_setting( name = "k8", values = {"host_cpu": "k8"}, From 576f0cb036cc44da270017dac8897ec1d23b6762 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Mon, 25 Nov 2024 18:34:45 +0000 Subject: [PATCH 6/7] Remove maven repo for bazel itself --- MODULE.bazel | 122 ----------------------------------------- repositories.maven.bzl | 6 -- third_party/BUILD | 2 +- 3 files changed, 1 insertion(+), 129 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 526f3446a..c09fa1972 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -68,125 +68,3 @@ maven.install( ], ) use_repo(maven, "copybara_maven") - -# Copied from bazelbuild/bazel/MODULE.bazel, has to be kept in sync -# TODO(hsudhof): Vendor bazel's starlark package instead -maven.install( - artifacts = [ - - "com.github.ben-manes.caffeine:caffeine:3.0.5", - "com.github.kevinstern:software-and-algorithms:1.0", - "com.github.stephenc.jcip:jcip-annotations:1.0-1", - "com.google.api-client:google-api-client-gson:1.35.2", - "com.google.api-client:google-api-client:1.35.2", - "com.google.auth:google-auth-library-credentials:1.6.0", - "com.google.auth:google-auth-library-oauth2-http:1.6.0", - "com.google.auto.service:auto-service-annotations:1.0.1", - "com.google.auto.service:auto-service:1.0", - "com.google.auto.value:auto-value-annotations:1.9", - "com.google.auto.value:auto-value:1.8.2", - "com.google.auto:auto-common:1.2.1", - "com.google.code.findbugs:jsr305:3.0.2", - "com.google.code.gson:gson:2.9.0", - "com.google.code.java-allocation-instrumenter:java-allocation-instrumenter:3.3.0", - "com.google.errorprone:error_prone_annotation:2.23.0", - "com.google.errorprone:error_prone_annotations:2.23.0", - "com.google.errorprone:error_prone_check_api:2.23.0", - "com.google.errorprone:error_prone_core:2.23.0", - "com.google.errorprone:error_prone_type_annotations:2.23.0", - "com.google.flogger:flogger-system-backend:0.5.1", - "com.google.flogger:flogger:0.5.1", - "com.google.flogger:google-extensions:0.5.1", - "com.google.guava:failureaccess:1.0.1", - "com.google.guava:guava:33.0.0-jre", - "com.google.http-client:google-http-client-gson:1.42.0", - "com.google.http-client:google-http-client:1.42.0", - "com.google.j2objc:j2objc-annotations:1.3", - "com.google.turbine:turbine:0.5.0", - "com.ryanharter.auto.value:auto-value-gson-extension:1.3.1", - "com.ryanharter.auto.value:auto-value-gson-runtime:1.3.1", - "com.ryanharter.auto.value:auto-value-gson-factory:1.3.1", - "com.squareup:javapoet:1.12.0", - "commons-collections:commons-collections:3.2.2", - "commons-lang:commons-lang:2.6", - "io.github.java-diff-utils:java-diff-utils:4.12", - "io.grpc:grpc-api:1.48.1", - "io.grpc:grpc-auth:1.48.1", - "io.grpc:grpc-context:1.48.1", - "io.grpc:grpc-core:1.48.1", - "io.grpc:grpc-netty:1.48.1", - "io.grpc:grpc-protobuf-lite:1.48.1", - "io.grpc:grpc-protobuf:1.48.1", - "io.grpc:grpc-stub:1.48.1", - "io.netty:netty-buffer:4.1.93.Final", - "io.netty:netty-codec-http2:4.1.93.Final", - "io.netty:netty-codec-http:4.1.93.Final", - "io.netty:netty-codec:4.1.93.Final", - "io.netty:netty-common:4.1.93.Final", - "io.netty:netty-handler-proxy:4.1.93.Final", - "io.netty:netty-handler:4.1.93.Final", - "io.netty:netty-resolver-dns:4.1.93.Final", - "io.netty:netty-resolver:4.1.93.Final", - "io.netty:netty-tcnative-boringssl-static:jar:linux-aarch_64:2.0.56.Final", - "io.netty:netty-tcnative-boringssl-static:jar:linux-x86_64:2.0.56.Final", - "io.netty:netty-tcnative-boringssl-static:jar:osx-aarch_64:2.0.56.Final", - "io.netty:netty-tcnative-boringssl-static:jar:osx-x86_64:2.0.56.Final", - "io.netty:netty-tcnative-boringssl-static:jar:windows-x86_64:2.0.56.Final", - "io.netty:netty-tcnative-classes:2.0.56.Final", - "io.netty:netty-transport-classes-epoll:4.1.93.Final", - "io.netty:netty-transport-classes-kqueue:4.1.93.Final", - "io.netty:netty-transport-native-epoll:jar:linux-aarch_64:4.1.93.Final", - "io.netty:netty-transport-native-epoll:jar:linux-x86_64:4.1.93.Final", - "io.netty:netty-transport-native-kqueue:jar:osx-aarch_64:4.1.93.Final", - "io.netty:netty-transport-native-kqueue:jar:osx-x86_64:4.1.93.Final", - "io.netty:netty-transport-native-unix-common:4.1.93.Final", - "io.netty:netty-transport-native-unix-common:jar:linux-aarch_64:4.1.93.Final", - "io.netty:netty-transport-native-unix-common:jar:linux-x86_64:4.1.93.Final", - "io.netty:netty-transport-native-unix-common:jar:osx-aarch_64:4.1.93.Final", - "io.netty:netty-transport-native-unix-common:jar:osx-x86_64:4.1.93.Final", - "io.netty:netty-transport:4.1.93.Final", - "io.reactivex.rxjava3:rxjava:3.1.2", - "it.unimi.dsi:fastutil:7.2.1", - "javax.activation:javax.activation-api:1.2.0", - "javax.annotation:javax.annotation-api:1.3.2", - "javax.inject:javax.inject:1", - "net.bytebuddy:byte-buddy-agent:1.14.5", - "net.bytebuddy:byte-buddy:1.14.5", - "org.apache.commons:commons-compress:1.20", - "org.apache.commons:commons-pool2:2.8.0", - "org.apache.tomcat:tomcat-annotations-api:8.0.5", - "org.apache.velocity:velocity:1.7", - "org.checkerframework:checker-qual:3.19.0", - "org.jcommander:jcommander:2.0", - "org.ow2.asm:asm-analysis:9.2", - "org.ow2.asm:asm-commons:9.2", - "org.ow2.asm:asm-tree:9.2", - "org.ow2.asm:asm-util:9.2", - "org.ow2.asm:asm:9.2", - "org.pcollections:pcollections:3.1.4", - "org.threeten:threeten-extra:1.5.0", - "org.tukaani:xz:1.9", - "org.yaml:snakeyaml:1.28", - "tools.profiler:async-profiler:3.0", - # The following jars are for testing. - # junit is not test only due to //src/java_tools/junitrunner/java/com/google/testing/junit/junit4:runner, - # and hamcrest is a dependency of junit. - "junit:junit:4.13.2", - "org.hamcrest:hamcrest-core:1.3", - ], - excluded_artifacts = [ - # org.apache.httpcomponents and org.eclipse.jgit:org.eclipse.jgit - # require java.security.jgss module to be embedded in the Bazel binary. - "org.apache.httpcomponents:httpclient", - "org.apache.httpcomponents:httpcore", - "org.eclipse.jgit:org.eclipse.jgit", - # We build protobuf Java library from source, exclude protobuf jars to be safe. - "com.google.protobuf:protobuf-java", - "com.google.protobuf:protobuf-javalite", - ], - repositories = [ - "https://repo1.maven.org/maven2", - ], - strict_visibility = True, -) -use_repo(maven, "maven") diff --git a/repositories.maven.bzl b/repositories.maven.bzl index c52808dff..5e01022ca 100644 --- a/repositories.maven.bzl +++ b/repositories.maven.bzl @@ -70,9 +70,3 @@ def copybara_maven_repositories(): artifacts = COPYBARA_MAVEN_ARTIFACTS, repositories = COPYBARA_MAVEN_ARTIFACT_ADDITIONAL_REPOSITORIES + ["https://repo1.maven.org/maven2"], ) - maybe( - maven_install, - name = "maven", - artifacts = COPYBARA_MAVEN_ARTIFACTS, - repositories = COPYBARA_MAVEN_ARTIFACT_ADDITIONAL_REPOSITORIES + ["https://repo1.maven.org/maven2"], - ) diff --git a/third_party/BUILD b/third_party/BUILD index 521457b56..c02e01ec6 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -67,7 +67,7 @@ java_library( java_library( name = "jcommander", exports = [ - "@maven//:org_jcommander_jcommander", + "@copybara_maven//:org_jcommander_jcommander", ], ) From 193426386ffb1e766d676b8236c9020d31b5bbbc Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Mon, 25 Nov 2024 18:39:16 +0000 Subject: [PATCH 7/7] Remove io_bazel repository --- MODULE.bazel | 3 --- repositories.bzl | 42 ------------------------------------------ third_party/bazel.bzl | 19 ------------------- 3 files changed, 64 deletions(-) delete mode 100644 third_party/bazel.bzl diff --git a/MODULE.bazel b/MODULE.bazel index c09fa1972..25d2cb141 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,9 +13,6 @@ bazel_dep(name = "rules_license", version = "0.0.8") bazel_dep(name = "rules_pkg", version = "0.10.1") bazel_dep(name = "rules_python", version = "0.31.0") -non_module_deps = use_extension("//:repositories.bzl", "non_module_deps") -use_repo(non_module_deps, "io_bazel") - maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven") maven.install( name = "copybara_maven", diff --git a/repositories.bzl b/repositories.bzl index 3b5dd87fc..6199d0f28 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -14,7 +14,6 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") -load("//third_party:bazel.bzl", "bazel_sha256", "bazel_version") load("//third_party:bazel_skylib.bzl", "skylib_sha256", "skylib_version") def copybara_repositories(): @@ -50,33 +49,6 @@ def copybara_repositories(): url = "https://github.com/bazelbuild/bazel-skylib/archive/" + skylib_version + ".zip", ) - EXPORT_WORKSPACE_IN_BUILD_FILE = [ - "test -f BUILD && chmod u+w BUILD || true", - "echo >> BUILD", - "echo 'exports_files([\"WORKSPACE\"], visibility = [\"//visibility:public\"])' >> BUILD", - ] - - EXPORT_WORKSPACE_IN_BUILD_FILE_WIN = [ - "Add-Content -Path BUILD -Value \"`nexports_files([`\"WORKSPACE`\"], visibility = [`\"//visibility:public`\"])`n\" -Force", - ] - - # Stuff used by Bazel Starlark syntax package transitively: - # LICENSE: The Apache Software License, Version 2.0 - maybe( - http_archive, - name = "com_google_protobuf", - patch_args = ["-p1"], - patches = ["@io_bazel//third_party/protobuf:21.7.patch"], - patch_cmds = EXPORT_WORKSPACE_IN_BUILD_FILE, - patch_cmds_win = EXPORT_WORKSPACE_IN_BUILD_FILE_WIN, - sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae", - strip_prefix = "protobuf-21.7", - urls = [ - "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", - "https://github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", - ], - ) - # Stuff used by Buildifier transitively: # LICENSE: The Apache Software License, Version 2.0 maybe( @@ -161,17 +133,3 @@ def copybara_repositories(): "http://github.com/keith/buildifier-prebuilt/archive/6.4.0.tar.gz", ], ) - - _non_module_deps(None) - -def _non_module_deps(_): - # LICENSE: The Apache Software License, Version 2.0 - maybe( - http_archive, - name = "io_bazel", - sha256 = bazel_sha256, - strip_prefix = "bazel-" + bazel_version, - url = "https://github.com/bazelbuild/bazel/archive/" + bazel_version + ".zip", - ) - -non_module_deps = module_extension(implementation = _non_module_deps) diff --git a/third_party/bazel.bzl b/third_party/bazel.bzl deleted file mode 100644 index d1297a822..000000000 --- a/third_party/bazel.bzl +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2019 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# This file is autogenerated by copybara, please do not edit. - -bazel_version = "3edda22913d1d42fb112c014fbb68f30993f5c18" -bazel_sha256 = "294dc13f51749c7654a063e31a57eff02e0777fc26ea7b224e5468fb292e7768"