Skip to content

wip: 0.5.0 - scorer updates and project scoped judgeval #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
abhishekg999 wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

wip: 0.5.0 - scorer updates and project scoped judgeval #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bf90640
wip
abhishekg999 Feb 9, 2026
c7401fc
chore patterns
abhishekg999 Feb 10, 2026
697f2f0
format
abhishekg999 Feb 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,4 +162,6 @@ $RECYCLE.BIN/
.env*


.idea
.idea

.metals
5 changes: 4 additions & 1 deletion .vscode/settings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,8 @@
"java.configuration.updateBuildConfiguration": "automatic",
"java.import.maven.enabled": true,
"java.compile.nullAnalysis.mode": "automatic",
"java.format.settings.url": ".vscode/java-formatter.xml"
"java.format.settings.url": ".vscode/java-formatter.xml",
"files.watcherExclude": {
"**/target": true
}
}
33 changes: 33 additions & 0 deletions bun.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion examples/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
<dependency>
<groupId>com.judgmentlabs</groupId>
<artifactId>judgeval-java</artifactId>
<version>0.4.2</version>
<version>0.5.0</version>
<exclusions>
<exclusion>
<groupId>io.opentelemetry</groupId>
Expand Down
8 changes: 5 additions & 3 deletions examples/src/main/java/examples/simple_chat/SimpleChat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,11 @@
public class SimpleChat {
public static void main(String[] args) {
var client = Judgeval.builder()
.projectName("SimpleChat-Java")
.apiKey(System.getenv("JUDGMENT_API_KEY"))
.organizationId(System.getenv("JUDGMENT_ORG_ID"))
.build();
var tracer = client.tracer().create().projectName("SimpleChat-Java").build();
var tracer = client.tracer().create().build();

OpenAIClient otelClient = OpenAIOkHttpClient.fromEnv();

Expand All @@ -36,7 +37,8 @@ public static void main(String[] args) {
.property("expected_output", "4")
.build());

tracer.asyncTraceEvaluate(client.scorers().tracePromptScorer().get("ExampleTraceScorer"));
tracer.asyncTraceEvaluate(
client.scorers().tracePromptScorer().get("ExampleTraceScorer"));

});

Expand All @@ -45,4 +47,4 @@ public static void main(String[] args) {
} catch (InterruptedException ignored) {
}
}
}
}
2 changes: 1 addition & 1 deletion judgeval-java/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.judgmentlabs</groupId>
<artifactId>judgeval-java</artifactId>
<version>0.4.2</version>
<version>0.5.0</version>
<packaging>jar</packaging>
<name>Judgeval Java</name>
<description>Java SDK for Judgeval</description>
Expand Down
53 changes: 49 additions & 4 deletions judgeval-java/src/main/java/com/judgmentlabs/judgeval/Judgeval.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
package com.judgmentlabs.judgeval;

import java.util.Objects;
import java.util.Optional;

import com.judgmentlabs.judgeval.evaluation.EvaluationFactory;
import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient;
import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectRequest;
import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectResponse;
import com.judgmentlabs.judgeval.scorers.ScorersFactory;
import com.judgmentlabs.judgeval.tracer.TracerFactory;
import com.judgmentlabs.judgeval.utils.Logger;

/**
* Main entry point for the Judgment SDK. Provides access to tracer, scorer, and
Expand All @@ -15,13 +19,22 @@ public class Judgeval {
private final String apiKey;
private final String organizationId;
private final String apiUrl;
private final String projectName;
private final Optional<String> projectId;
private final JudgmentSyncClient internalClient;

protected Judgeval(Builder builder) {
this.apiKey = Objects.requireNonNull(builder.apiKey, "apiKey required");
this.organizationId = Objects.requireNonNull(builder.organizationId, "organizationId required");
this.apiUrl = builder.apiUrl != null ? builder.apiUrl : Env.JUDGMENT_API_URL;
this.projectName = Objects.requireNonNull(builder.projectName, "projectName required");
this.apiUrl = Optional.ofNullable(builder.apiUrl).orElse(Env.JUDGMENT_API_URL);
this.internalClient = new JudgmentSyncClient(apiUrl, apiKey, organizationId);
this.projectId = resolveProjectId(projectName);

if (projectId.isEmpty()) {
Logger.warning("Project '" + projectName + "' not found. "
+ "Some operations requiring project_id will be skipped.");
}
}

/**
Expand All @@ -30,7 +43,7 @@ protected Judgeval(Builder builder) {
* @return the tracer factory
*/
public TracerFactory tracer() {
return new TracerFactory(internalClient);
return new TracerFactory(internalClient, projectName, projectId);
}

/**
Expand All @@ -39,7 +52,7 @@ public TracerFactory tracer() {
* @return the scorer factory
*/
public ScorersFactory scorers() {
return new ScorersFactory(internalClient);
return new ScorersFactory(internalClient, projectId);
}

/**
Expand All @@ -48,7 +61,15 @@ public ScorersFactory scorers() {
* @return the evaluation factory
*/
public EvaluationFactory evaluation() {
return new EvaluationFactory(internalClient);
return new EvaluationFactory(internalClient, projectId, projectName);
}

public Optional<String> getProjectId() {
return projectId;
}

public String getProjectName() {
return projectName;
}

/**
Expand All @@ -60,13 +81,37 @@ public static Builder builder() {
return new Builder();
}

private Optional<String> resolveProjectId(String name) {
try {
ResolveProjectRequest request = new ResolveProjectRequest();
request.setProjectName(name);
ResolveProjectResponse response = internalClient.postProjectsResolve(request);
return Optional.ofNullable(response.getProjectId()).map(Object::toString);
} catch (Exception e) {
return Optional.empty();
}
Comment on lines +90 to +92
Copy link

@gemini-code-assist gemini-code-assist bot Feb 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Catching a generic Exception and returning an empty Optional without logging can hide important errors during project ID resolution, such as network issues or authentication failures. This makes debugging difficult. It's recommended to log the exception to provide visibility into what went wrong.

        } catch (Exception e) {
            Logger.error("Failed to resolve project ID for project '" + name + "'.", e);
            return Optional.empty();
        }

}

/**
* Builder for configuring and creating Judgeval instances.
*/
public static class Builder {
private String apiKey = Env.JUDGMENT_API_KEY;
private String organizationId = Env.JUDGMENT_ORG_ID;
private String apiUrl = Env.JUDGMENT_API_URL;
private String projectName;

/**
* Sets the project name used for project-scoped operations.
*
* @param projectName
* the project name
* @return this builder
*/
public Builder projectName(String projectName) {
this.projectName = projectName;
return this;
}

/**
* Sets the API key for authentication.
Expand Down
Loading