diff --git a/dance.py b/dance.py index f12793e..7211384 100644 --- a/dance.py +++ b/dance.py @@ -5,6 +5,15 @@ from statistics import mean from typing import List, Tuple +from datetime import datetime + +from together import Together + +from composio_crewai import App, ComposioToolSet +from crewai import Agent, Task, Crew +from dotenv import load_dotenv +from langchain_openai import ChatOpenAI + import cv2 import numpy as np import mediapipe as mp @@ -20,6 +29,9 @@ mp_draw = mp.solutions.drawing_utils mp_pose = mp.solutions.pose +# Initialize variables to keep track of the highest average difference and corresponding limb +max_diff = None #-float('inf') +max_limb = None class PoseLandmark(Enum): NOSE = 0 @@ -51,6 +63,46 @@ class PoseLandmark(Enum): ] +# Create and Execute Agent. +def followup_event(): + + # Initialize the language model + llm = ChatOpenAI(model="gpt-4o") + + # Define tools for the agents + # We are using Google calendar tool from composio to connect to our calendar account. + composio_toolset = ComposioToolSet() + tools = composio_toolset.get_tools(apps=[App.GOOGLECALENDAR]) + + # Retrieve the current date and time + date = datetime.today().strftime("%Y-%m-%d+1") + timezone = datetime.now().astimezone().tzinfo + + # Setup Todo + followup = """ + 6PM - 7PM -> Meeting for dance recital followup, + 10:30PM - 11PM -> Dance Practice + """ + calendar_agent = Agent( + role="Google Calendar Agent", + goal="""You take action on Google Calendar using Google Calendar APIs""", + backstory="""You are an AI agent responsible for taking actions on Google Calendar on users' behalf. + You need to take action on Calendar using Google Calendar APIs. Use correct tools to run APIs from the given tool-set.""", + verbose=True, + tools=tools, + llm=llm, + cache=False, + ) + task = Task( + description=f"Book slots according to {followup}. Label them with the work provided to be done in that time period. Schedule it for today. Today's date is {date} (it's in YYYY-MM-DD format) and make the timezone be {timezone}.", + agent=calendar_agent, + expected_output="if free slot is found", + ) + crew = Crew(agents=[calendar_agent], tasks=[task]) + result = crew.kickoff() + print(result) + return "Crew run initiated", 200 + def get_video_duration(filename: str) -> float: """Returns the duration of a video clip in seconds.""" video = cv2.VideoCapture(filename) @@ -92,6 +144,19 @@ def extract_landmarks(video_path: str) -> Tuple[List[List[Tuple[float, float]]], return xy_landmark_coords, frames, landmarks +def generate_feedback_summary(ref_angles: List[List[float]], comp_angles: List[List[float]]): + """Generates a summary of feedback on limb angles for all frames.""" + feedback_summary = {} + num_frames = len(ref_angles) + + for limb_idx, (start, end) in enumerate(LIMB_CONNECTIONS): + diffs = [abs(ref_angles[frame_idx][limb_idx] - comp_angles[frame_idx][limb_idx]) + for frame_idx in range(num_frames)] + avg_diff = mean(diffs) + feedback_summary[f"Limb {start.name} to {end.name}"] = avg_diff + + return feedback_summary + def calculate_limb_angles(frame_landmarks: List[List[Tuple[float, float]]]) -> List[List[float]]: """Calculates limb angles for each frame.""" @@ -132,6 +197,10 @@ def compare_dancers(ref_landmarks: List[List[Tuple[float, float]]], ref_pose_results: List, comp_pose_results: List) -> float: """Compares two dancers and returns a synchronization score.""" + + #global variables + + global max_diff, max_limb # get number of comparable frames num_frames = min(len(ref_landmarks), len(comp_landmarks)) @@ -142,42 +211,98 @@ def compare_dancers(ref_landmarks: List[List[Tuple[float, float]]], out_of_sync_frames = 0 score = 100.0 - print("Analysing dancers...") + # Dictionary to store frame indices and their average differences + frame_errors = {} + + print("Analyzing dancers...") video_writer = cv2.VideoWriter(f'{OUTPUT_DIR}/output.mp4', cv2.VideoWriter_fourcc(*'mp4v'), FPS, (2 * 720, 1280)) + # Define colors to flash through + colors = [(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0), (255, 0, 255), (0, 255, 255)] + for frame_idx in range(num_frames): - # difference in angle for each limb - frame_diffs = [abs(ref_angles[frame_idx][j] - comp_angles[frame_idx][j]) / 180 for j in - range(len(LIMB_CONNECTIONS))] + frame_diffs = [abs(ref_angles[frame_idx][j] - comp_angles[frame_idx][j]) / 180 for j in range(len(LIMB_CONNECTIONS))] frame_diff = mean(frame_diffs) + # Store the frame index and its average difference + frame_errors[frame_idx] = frame_diff + ref_frame = ref_frames[frame_idx] comp_frame = comp_frames[frame_idx] - # annotation skeleton and score on the frame mp_draw.draw_landmarks(ref_frame, ref_pose_results[frame_idx].pose_landmarks, mp_pose.POSE_CONNECTIONS) mp_draw.draw_landmarks(comp_frame, comp_pose_results[frame_idx].pose_landmarks, mp_pose.POSE_CONNECTIONS) display = np.concatenate((ref_frame, comp_frame), axis=1) color = (0, 0, 255) if frame_diff > SYNC_THRESHOLD else (255, 0, 0) - cv2.putText(display, f"Diff: {frame_diff:.2f}", (40, 40), cv2.FONT_HERSHEY_SIMPLEX, 1, color, 3) - # determine if synced if frame_diff > SYNC_THRESHOLD: out_of_sync_frames += 1 score = ((frame_idx + 1 - out_of_sync_frames) / (frame_idx + 1)) * 100.0 - cv2.putText(display, f"Score: {score:.2f}%", (ref_frame.shape[1] + 40, 40), cv2.FONT_HERSHEY_SIMPLEX, 1, color, - 3) + cv2.putText(display, f"Score: {score:.2f}%", (ref_frame.shape[1] + 40, 40), cv2.FONT_HERSHEY_SIMPLEX, 1, color, 3) + + # --- Added: Center the "MindsDB Agents Hackathon LFG!!!" text and flash different colors --- + text = "MindsDB Agents Hackathon LFG!!!" + font = cv2.FONT_HERSHEY_SIMPLEX + font_scale = 1.5 + thickness = 3 + text_size = cv2.getTextSize(text, font, font_scale, thickness)[0] + + # Calculate the center position + text_x = (display.shape[1] - text_size[0]) // 2 # x-coordinate for centering + text_y = (display.shape[0] * 7 // 8) # Moves the text 50% lower + + # Pick a color based on the frame index (modulo to cycle through colors) + flashing_color = colors[frame_idx % len(colors)] # --- Change: Cycle through colors --- + + # Put the text at the center of the frame with the flashing color + cv2.putText(display, text, (text_x, text_y), font, font_scale, flashing_color, thickness) # --- Change: Use flashing_color --- + + # Show the display and write it to the output video cv2.imshow(str(frame_idx), display) video_writer.write(display) cv2.waitKey(1) video_writer.release() - return score + + # Calculate average limb difference for each 5-second window + frame_rate = int(FPS) + window_size = int(5 * frame_rate) + window_errors = [] + + for start_idx in range(0, num_frames, window_size): + end_idx = min(start_idx + window_size, num_frames) + window_diff = [frame_errors[idx] for idx in range(start_idx, end_idx)] + avg_window_diff = mean(window_diff) + + start_time = start_idx / frame_rate + end_time = end_idx / frame_rate + + window_errors.append((start_time, end_time, avg_window_diff)) + + sorted_windows = sorted(window_errors, key=lambda x: x[2], reverse=True) + + print("\n5-Second Time Ranges with the Most Errors:") + for start_time, end_time, avg_diff in sorted_windows[:5]: + print(f"Range {start_time:.2f}s - {end_time:.2f}s: {avg_diff:.2f} degrees difference on average") + + feedback_summary = generate_feedback_summary(ref_angles, comp_angles) + print("\nDetailed Feedback:") + + for limb, avg_diff in feedback_summary.items(): + print(f"{limb}: {avg_diff:.2f} degrees difference on average") + + if max_diff is None or avg_diff > max_diff: + max_diff = avg_diff + max_limb = limb + + + + return score, max_diff, max_limb def convert_to_same_framerate(clip: str) -> str: @@ -228,7 +353,6 @@ def trim_clips(ref_clip: str, comparison_clip: str, offset: float) -> Tuple[str, return ref_cut, comp_cut - def main(ref_clip: str, comparison_clip: str, compare_only: bool = False): # ensure output directory exists os.makedirs(OUTPUT_DIR, exist_ok=True) @@ -258,10 +382,55 @@ def main(ref_clip: str, comparison_clip: str, compare_only: bool = False): comp_landmarks, comp_frames, comp_pose_results = extract_landmarks(comp_cut) # compare - score = compare_dancers(ref_landmarks, comp_landmarks, ref_frames, comp_frames, ref_pose_results, comp_pose_results) + score, max_diff, max_limb = compare_dancers(ref_landmarks, comp_landmarks, ref_frames, comp_frames, ref_pose_results, comp_pose_results) print(f"\nYou are {score:.2f}% in sync with your model dancer!") + + + if score < 60: + prompt = f"What are some ways to improve my dance move accuracy since I have an accuracy of {score:.2f}%? Mention the limb with the highest average difference is {max_limb} with an average difference of {max_diff:.2f} degrees. Please make sure to mention my score, my style of dance which are Boogalo and Funk, and include a brief history of each dance style." + # Print the values to verify they are returned correctly + print(f"Score: {score:.2f}") + print(f"Limb with highest average difference: {max_limb}") + print(f"Average difference for the limb: {max_diff:.2f}") + print(prompt) + # Call AI agent to add followup dance practice events to improve + followup_event() + client = Together(api_key=os.environ.get("34433f9db54ef4d633660b7961f354a8e6a6ee1094d366dad96c16ffe8558cd0")) + # Create a streaming completion request + stream = client.chat.completions.create( + model="meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + messages=[{"role": "user", "content": prompt}], + stream=True, + ) + + for chunk in stream: + print(chunk.choices[0].delta.content or "", end="", flush=True) + + else: + prompt = f"Say congratulations and we are ready to start promoting your video, mention the score {score:.2f}%? Mention the limb with the highest average difference is {max_limb} with an average difference of {max_diff:.2f} degrees. Please make sure to mention my score, my style of dance which are Boogalo and Funk, and include a brief history of each dance style. you could also try a similar dance style like waving next. You could try roboting as well. Mention all this." + print(f"Score: {score:.2f}") + print(f"Limb with highest average difference: {max_limb}") + print(f"Average difference for the limb: {max_diff:.2f}") + print(f"Great job at being above 60%") + print(prompt) + # Call AI agent to add followup dance practice events to improve + followup_event() + client = Together(api_key=os.environ.get("34433f9db54ef4d633660b7961f354a8e6a6ee1094d366dad96c16ffe8558cd0")) + # Create a streaming completion request + stream = client.chat.completions.create( + model="meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + messages=[{"role": "user", "content": prompt}], + stream=True, + ) + + for chunk in stream: + print(chunk.choices[0].delta.content or "", end="", flush=True) + + + + if __name__ == "__main__": import sys @@ -277,3 +446,5 @@ def main(ref_clip: str, comparison_clip: str, compare_only: bool = False): compare_only = len(sys.argv) > 3 and sys.argv[3] == '--compare-only' main(ref_clip, comparison_clip, compare_only) + + diff --git a/followup_event.py b/followup_event.py new file mode 100644 index 0000000..a25f4d1 --- /dev/null +++ b/followup_event.py @@ -0,0 +1,57 @@ +# Import base packages +import os +from datetime import datetime + +from composio_crewai import App, ComposioToolSet +from crewai import Agent, Task, Crew +from dotenv import load_dotenv +from langchain_openai import ChatOpenAI + + +# Load environment variables +load_dotenv() + +# Initialize the language model +llm = ChatOpenAI(model="gpt-4o") + +# Define tools for the agents +# We are using Google calendar tool from composio to connect to our calendar account. +composio_toolset = ComposioToolSet() +tools = composio_toolset.get_tools(apps=[App.GOOGLECALENDAR]) + +# Retrieve the current date and time +date = datetime.today().strftime("%Y-%m-%d+1") +timezone = datetime.now().astimezone().tzinfo + +# Setup Todo +followup = """ + 6PM - 7PM -> Meeting for dance recital followup, + 10:30PM - 11PM -> Dance Practice, + 8PM - 10PM -> Dinner with promoter +""" + + +# Create and Execute Agent. +def followup_event(): + calendar_agent = Agent( + role="Google Calendar Agent", + goal="""You take action on Google Calendar using Google Calendar APIs""", + backstory="""You are an AI agent responsible for taking actions on Google Calendar on users' behalf. + You need to take action on Calendar using Google Calendar APIs. Use correct tools to run APIs from the given tool-set.""", + verbose=True, + tools=tools, + llm=llm, + cache=False, + ) + task = Task( + description=f"Book slots according to {followup}. Label them with the work provided to be done in that time period. Schedule it for today. Today's date is {date} (it's in YYYY-MM-DD format) and make the timezone be {timezone}.", + agent=calendar_agent, + expected_output="if free slot is found", + ) + crew = Crew(agents=[calendar_agent], tasks=[task]) + result = crew.kickoff() + print(result) + return "Crew run initiated", 200 + + +followup_event() diff --git a/nat2.mp4 b/nat2.mp4 new file mode 100644 index 0000000..170c318 Binary files /dev/null and b/nat2.mp4 differ diff --git a/nat3.mp4 b/nat3.mp4 new file mode 100644 index 0000000..e56ddda Binary files /dev/null and b/nat3.mp4 differ diff --git a/requirements.txt b/requirements.txt index 6f52475..c49d04f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -mediapipe-silicon==0.8.11 +mediapipe-silicon==0.9.0.1 opencv-python==4.6.0.66 diff --git a/video/kern.mov b/video/kern.mov new file mode 100644 index 0000000..7edb89e Binary files /dev/null and b/video/kern.mov differ diff --git a/video/nat.mov b/video/nat.mov new file mode 100644 index 0000000..4c41686 Binary files /dev/null and b/video/nat.mov differ diff --git a/video/nat1.mov b/video/nat1.mov new file mode 100644 index 0000000..312b8ab Binary files /dev/null and b/video/nat1.mov differ