Ai rubric feedback

apps/backend/.template.env

@@ -27,4 +27,11 @@ SCHEDULER_URL="http://localhost:8000"
 DIGITALOCEAN_ENDPOINT=""
 DIGITALOCEAN_SPACE=""
 DIGITALOCEAN_KEY=""
-DIGITALOCEAN_SECRET=""
+DIGITALOCEAN_SECRET=""
+
+# AI provider
+LANGSMITH_TRACING=true
+LANGSMITH_ENDPOINT="https://api.smith.langchain.com"
+LANGSMITH_API_KEY=""
+LANGSMITH_PROJECT="lems"
+OPENAI_API_KEY=""

apps/backend/src/lib/ai/agents/feedback/agent.ts

@@ -0,0 +1,44 @@
+import { z } from 'zod';
+import { START, StateGraph, END } from '@langchain/langgraph';
+import { GraphAnnotation, RubricSchema } from './types';
+import { checkFeedbackValidity } from './handlers/check-feedback-validity';
+import { improveSpellingAndGrammar } from './handlers/improve-spelling-and-grammar';
+import { improvePhrasing } from './handlers/improve-phrasing';
+
+const graph = new StateGraph(GraphAnnotation)
+  .addNode('checkFeedbackValidity', checkFeedbackValidity)
+  .addNode('improveSpellingAndGrammar', improveSpellingAndGrammar)
+  .addNode('improvePhrasing', improvePhrasing);
+
+const continueIfValid = (state: typeof GraphAnnotation.State) => {
+  return state.isValid ? 'improveSpellingAndGrammar' : END;
+};
+
+graph
+  .addEdge(START, 'checkFeedbackValidity')
+  .addConditionalEdges('checkFeedbackValidity', continueIfValid)
+  .addEdge('improveSpellingAndGrammar', 'improvePhrasing')
+  .addEdge('improvePhrasing', END);
+
+const workflow = graph.compile();
+
+type AgentInput = z.infer<typeof RubricSchema>;
+export async function enhanceFeedback(rubric: AgentInput) {
+  try {
+    const result = await workflow.invoke(
+      {
+        rubric,
+        feedback: rubric.feedback || { greatJob: '', thinkAbout: '' },
+        isValid: true,
+        error: ''
+      },
+      {
+        runName: 'enhance-feedback'
+      }
+    );
+
+    return result;
+  } catch (error) {
+    console.error('Error enhancing feedback:', error);
+  }
+}

apps/backend/src/lib/ai/agents/feedback/handlers/check-feedback-validity.ts

@@ -0,0 +1,50 @@
+import { SystemMessage, HumanMessage } from '@langchain/core/messages';
+import { GraphAnnotation } from '../types';
+import { llm } from '../../../llm';
+
+export const checkFeedbackValidity = async (state: typeof GraphAnnotation.State) => {
+  const analysisPrompt = `
+    Analyze the following feedback for a FIRST LEGO League team rubric and determine if it's problematic.
+    Problematic feedback is defined as:
+    1. Empty or extremely short feedback that provides no real insight (fewer than 5 meaningful words)
+    2. Content that is unclear, incoherent, or cannot be meaningfully improved
+
+    Great Job Feedback: ${state.feedback.greatJob || '(empty)'}
+    Think About Feedback: ${state.feedback.thinkAbout || '(empty)'}
+
+    First analyze the sentiment and content of each feedback section.
+    Then determine if either section is problematic based on the criteria above.
+    Return a JSON object with { "isProblematic": boolean, "reason": string if problematic }
+    `;
+
+  const response = await llm.invoke([
+    new SystemMessage(
+      'You are a feedback quality analysis assistant. Analyze feedback critically and objectively.'
+    ),
+    new HumanMessage(analysisPrompt),
+    new SystemMessage(
+      "You MUST return a JSON object with the keys 'isProblematic' and 'reason' if applicable. \
+        Do not include any other text or explanations. The value of \"reason\" should be a string explaining the problem."
+    )
+  ]);
+
+  const content = response.content.toString();
+  let result: { isProblematic: boolean; reason?: string };
+
+  try {
+    const jsonMatch = content.match(/{[\s\S]*}/);
+    if (jsonMatch) {
+      result = JSON.parse(jsonMatch[0]);
+    } else {
+      throw new Error('Invalid JSON format in LLM response');
+    }
+  } catch (error) {
+    console.error('Error parsing feedback analysis:', error);
+    result = { isProblematic: true, reason: 'Invalid response format from LLM' };
+  }
+
+  return {
+    isValid: !result.isProblematic,
+    error: result.reason || ''
+  };
+};

apps/backend/src/lib/ai/agents/feedback/handlers/improve-phrasing.ts

@@ -0,0 +1,66 @@
+import { SystemMessage, HumanMessage } from '@langchain/core/messages';
+import { GraphAnnotation } from '../types';
+import { rubricsSchemas } from '@lems/season';
+import { llm } from '../../../llm';
+
+export const improvePhrasing = async (state: typeof GraphAnnotation.State) => {
+  const { rubric, feedback } = state;
+
+  const systemPrompt = `You are an AI assistant designed to help improve feedback on team rubrics for a FIRST LEGO League competition.
+  Your goal is to enhance feedback clarity while preserving the original feedback's intent and emotion.
+
+  The feedback consists of two parts:
+  1. "greatJob" - Positive feedback highlighting what the team did well
+  2. "thinkAbout" - Constructive feedback suggesting areas for improvement
+
+  If the feedback is already good, return it as is without any changes.
+  If the feedback is in Hebrew (עברית), you must maintain it in Hebrew. Do NOT translate to any other language.
+  `;
+
+  const context = `
+    Rubric Category: ${rubric.category}
+    Rubric Scores: ${JSON.stringify(rubric.values, null, 2)}  
+
+    Localized rubric: ${JSON.stringify(rubricsSchemas[rubric.category], null, 2)}`;
+
+  const greatJobPrompt = `
+    Improve the phrasing of the following "Great Job" feedback to be clearer and more concise, while preserving the original meaning and emotion. 
+    Only enhance phrasing that needs improvement; preserve good parts. 
+    Use rubric terminology and avoid generic phrases.
+
+    Context:
+    ${context}
+
+    Original Feedback (with grammar and spelling fixed):
+    ${feedback.greatJob}
+    `;
+
+  const thinkAboutPrompt = `
+    Improve the phrasing of the following "Think About" feedback to be clearer, more constructive, and more concise, 
+    while preserving the original meaning and emotion. Only enhance phrasing that needs improvement; preserve good parts. 
+    Use rubric terminology and avoid generic phrases.
+
+    Context:
+    ${context}
+
+    Original Feedback (with grammar and spelling fixed):
+    ${feedback.thinkAbout}
+    `;
+
+  const greatJobResponse = await llm.invoke([
+    new SystemMessage(systemPrompt),
+    new HumanMessage(greatJobPrompt)
+  ]);
+
+  const thinkAboutResponse = await llm.invoke([
+    new SystemMessage(systemPrompt),
+    new HumanMessage(thinkAboutPrompt)
+  ]);
+
+  return {
+    feedback: {
+      greatJob: greatJobResponse.content.toString(),
+      thinkAbout: thinkAboutResponse.content.toString()
+    }
+  };
+};

apps/backend/src/lib/ai/agents/feedback/handlers/improve-spelling-and-grammar.ts

@@ -0,0 +1,43 @@
+import { SystemMessage, HumanMessage } from '@langchain/core/messages';
+import { GraphAnnotation } from '../types';
+import { llm } from '../../../llm';
+
+export const improveSpellingAndGrammar = async (state: typeof GraphAnnotation.State) => {
+  const { feedback } = state;
+
+  const greatJobPrompt = `
+    Fix spelling and grammatical errors in the following text. Only fix errors, don't change the meaning.
+    IMPORTANT: If the text is in Hebrew (עברית), you must maintain it in Hebrew. Do NOT translate to any other language.
+
+    ${feedback.greatJob}
+    `;
+
+  const thinkAboutPrompt = `
+    Fix spelling and grammatical errors in the following text. Only fix errors, don't change the meaning.
+    IMPORTANT: If the text is in Hebrew (עברית), you must maintain it in Hebrew. Do NOT translate to any other language.
+
+    ${feedback.thinkAbout}
+    `;
+
+  // Fix both feedback sections independently
+  const greatJobResponse = await llm.invoke([
+    new SystemMessage(
+      'You are a spelling and grammar correction assistant. You MUST preserve the original language of the text (Hebrew/עברית). DO NOT translate text into English or any other language.'
+    ),
+    new HumanMessage(greatJobPrompt)
+  ]);
+
+  const thinkAboutResponse = await llm.invoke([
+    new SystemMessage(
+      'You are a spelling and grammar correction assistant. You MUST preserve the original language of the text (Hebrew/עברית). DO NOT translate text into English or any other language.'
+    ),
+    new HumanMessage(thinkAboutPrompt)
+  ]);
+
+  return {
+    feedback: {
+      greatJob: greatJobResponse.content.toString(),
+      thinkAbout: thinkAboutResponse.content.toString()
+    }
+  };
+};

apps/backend/src/lib/ai/agents/feedback/types.ts

@@ -0,0 +1,36 @@
+import { Annotation } from '@langchain/langgraph';
+import { z } from 'zod';
+
+export const GraphAnnotation = Annotation.Root({
+  rubric: Annotation<z.infer<typeof RubricSchema>>(),
+  feedback: Annotation<z.infer<typeof FeedbackSchema>>(),
+
+  isValid: Annotation<boolean>(),
+  error: Annotation<string>()
+});
+
+const FeedbackSchema = z.object({
+  greatJob: z.string().describe('Positive feedback highlighting what the team did well'),
+  thinkAbout: z.string().describe('Constructive feedback suggesting areas for improvement')
+});
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+export const RubricSchema = z.object({
+  category: z.string().describe('The judging category for this rubric'),
+  feedback: FeedbackSchema,
+  values: z
+    .record(
+      z.object({
+        value: z.number().describe("A score between 1 and 4 indicating the team's performance"),
+        notes: z
+          .string()
+          .optional()
+          .describe(
+            'Optional note explaining why the team received the highest score (4) for this value'
+          )
+      })
+    )
+    .optional()
+    .describe('Rubric score values'),
+  awards: z.record(z.boolean()).optional().describe('Optional awards selections')
+});

apps/backend/src/lib/ai/llm.ts

@@ -0,0 +1,3 @@
+import { ChatOpenAI } from '@langchain/openai';
+
+export const llm = new ChatOpenAI({ model: 'gpt-4o', temperature: 0 });

apps/backend/src/main.ts

@@ -15,6 +15,7 @@ import dashboardRouter from './routers/dashboard/index';
 import websocket from './websocket/index';
 import wsAuth from './middlewares/websocket/auth';
 import wsValidateDivision from './middlewares/websocket/division-validator';
+import aiRouter from './routers/ai-router';
 
 const app = express();
 const server = http.createServer(app);
@@ -39,6 +40,7 @@ app.use('/auth', authRouter);
 app.use('/public', publicRouter);
 app.use('/dashboard', dashboardRouter);
 app.use('/api', apiRouter);
+app.use('/ai', aiRouter);
 
 app.get('/status', (req, res) => {
   res.status(200).json({ ok: true });

apps/backend/src/routers/ai-router.ts

@@ -0,0 +1,44 @@
+import { Router } from 'express';
+import { enhanceFeedback } from '../lib/ai/agents/feedback/agent';
+import asyncHandler from 'express-async-handler';
+import { getRubric } from '@lems/database';
+import { ObjectId } from 'mongodb';
+
+const router = Router();
+
+router.post(
+  '/enhance-feedback',
+  asyncHandler(async (req, res) => {
+    const rubric = await getRubric({ _id: new ObjectId('679a55f199042139ba459aa0') });
+    if (!rubric) {
+      res.status(404).json({ error: 'Rubric not found' });
+      return;
+    }
+    console.log('Enhancing feedback for rubric:', rubric.teamId, rubric.category);
+
+    try {
+      const { category, data } = rubric;
+      const { values, feedback, awards } = data;
+      const agentInput = {
+        category,
+        values,
+        feedback,
+        awards
+      };
+      const result = await enhanceFeedback(agentInput);
+      res.status(200).json({
+        message: 'Feedback enhancement process started successfully',
+        original: rubric.data?.feedback,
+        updated: result.feedback
+      });
+    } catch (error) {
+      console.error('Error enhancing feedback:', error);
+      res.status(500).json({
+        error: 'Failed to enhance feedback',
+        details: error instanceof Error ? error.message : 'Unknown error occurred'
+      });
+    }
+  })
+);
+
+export default router;