Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 14 additions & 89 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ license = "Apache-2.0"
repository = "https://github.com/PlatformNetwork/term-challenge"

[workspace.dependencies]
platform-challenge-sdk = { git = "https://github.com/PlatformNetwork/platform.git", features = ["http-server"] }
platform-core = { git = "https://github.com/PlatformNetwork/platform.git" }
platform-challenge-sdk = { git = "https://github.com/PlatformNetwork/platform-v2", branch = "main", features = ["http-server"] }
platform-core = { git = "https://github.com/PlatformNetwork/platform-v2", branch = "main" }
sp-core = { version = "31.0", default-features = false, features = ["std"] }
sled = "0.34"
bincode = "1.3"
Expand Down
2 changes: 1 addition & 1 deletion server/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ name = "term-challenge-server"
path = "src/main.rs"

[dependencies]
platform-challenge-sdk = { path = "/workspace/platform-v2/crates/challenge-sdk", features = ["http-server"] }
platform-challenge-sdk = { git = "https://github.com/PlatformNetwork/platform-v2", branch = "main", features = ["http-server"] }

axum = { version = "0.7", features = ["json"] }
tokio = { version = "1.40", features = ["full"] }
Expand Down
38 changes: 19 additions & 19 deletions server/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ use serde_json::json;
use tracing::info;

#[derive(Parser)]
#[command(name = "term-challenge-server", about = "Terminal Benchmark Challenge Server")]
#[command(
name = "term-challenge-server",
about = "Terminal Benchmark Challenge Server"
)]
struct Cli {
#[arg(long, env = "CHALLENGE_HOST", default_value = "0.0.0.0")]
host: String,
Expand Down Expand Up @@ -167,7 +170,11 @@ impl TerminalBenchChallenge {
})
.collect();

entries.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal));
entries.sort_by(|a, b| {
b.score
.partial_cmp(&a.score)
.unwrap_or(std::cmp::Ordering::Equal)
});
for (i, entry) in entries.iter_mut().enumerate() {
entry.rank = (i + 1) as u32;
}
Expand Down Expand Up @@ -307,10 +314,7 @@ impl ServerChallenge for TerminalBenchChallenge {
}
}

async fn evaluate(
&self,
req: EvaluationRequest,
) -> Result<EvaluationResponse, ChallengeError> {
async fn evaluate(&self, req: EvaluationRequest) -> Result<EvaluationResponse, ChallengeError> {
let submission: SubmissionData = serde_json::from_value(req.data.clone()).map_err(|e| {
ChallengeError::Evaluation(format!("Failed to parse submission data: {}", e))
})?;
Expand Down Expand Up @@ -341,18 +345,17 @@ impl ServerChallenge for TerminalBenchChallenge {
}

let total_tasks = submission.task_results.len() as f64;
let passed_tasks = submission
.task_results
.iter()
.filter(|r| r.passed)
.count() as f64;
let passed_tasks = submission.task_results.iter().filter(|r| r.passed).count() as f64;
let pass_rate = passed_tasks / total_tasks;

let avg_score: f64 =
submission.task_results.iter().map(|r| r.score).sum::<f64>() / total_tasks;

let total_execution_time_ms: u64 =
submission.task_results.iter().map(|r| r.execution_time_ms).sum();
let total_execution_time_ms: u64 = submission
.task_results
.iter()
.map(|r| r.execution_time_ms)
.sum();

let final_score = (pass_rate * 0.7 + avg_score * 0.3).clamp(0.0, 1.0);

Expand Down Expand Up @@ -392,10 +395,7 @@ impl ServerChallenge for TerminalBenchChallenge {
))
}

async fn validate(
&self,
req: ValidationRequest,
) -> Result<ValidationResponse, ChallengeError> {
async fn validate(&self, req: ValidationRequest) -> Result<ValidationResponse, ChallengeError> {
let mut errors = Vec::new();
let mut warnings = Vec::new();

Expand Down Expand Up @@ -505,8 +505,8 @@ async fn main() -> Result<()> {
Some(id) => ChallengeId::from_str(id)
.ok_or_else(|| anyhow::anyhow!("Invalid challenge ID UUID: {}", id))?,
None => {
let id = ChallengeId::from_str("a1b2c3d4-e5f6-7890-abcd-ef1234567890")
.unwrap_or_default();
let id =
ChallengeId::from_str("a1b2c3d4-e5f6-7890-abcd-ef1234567890").unwrap_or_default();
info!("No challenge ID provided, using default: {}", id);
id
}
Expand Down
2 changes: 1 addition & 1 deletion wasm/src/routes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ pub fn get_route_definitions() -> Vec<WasmRouteDefinition> {
method: String::from("POST"),
path: String::from("/submit"),
description: String::from("Submission endpoint: receives zip package and metadata"),
requires_auth: false,
requires_auth: true,
},
WasmRouteDefinition {
method: String::from("GET"),
Expand Down
19 changes: 1 addition & 18 deletions wasm/src/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -295,21 +295,4 @@ impl Default for WhitelistConfig {
}
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct LlmMessage {
pub role: String,
pub content: String,
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct LlmRequest {
pub model: String,
pub messages: Vec<LlmMessage>,
pub max_tokens: u32,
pub temperature: f64,
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct LlmResponse {
pub content: String,
}
pub use platform_challenge_sdk_wasm::{LlmMessage, LlmRequest, LlmResponse};