diff --git a/community_notebooks/rag_fiqa_mrr_optimization.ipynb b/community_notebooks/rag_fiqa_mrr_optimization.ipynb new file mode 100644 index 00000000..aa57772d --- /dev/null +++ b/community_notebooks/rag_fiqa_mrr_optimization.ipynb @@ -0,0 +1,753 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e4c769cc", + "metadata": { + "id": "e4c769cc" + }, + "source": [ + "### Install and Initialize RapidFire AI" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0e23e77", + "metadata": { + "id": "d0e23e77" + }, + "outputs": [], + "source": [ + "try:\n", + " import rapidfireai\n", + " print(\"✅ rapidfireai already installed\")\n", + "except ImportError:\n", + " !pip install rapidfireai==0.14.0 # Takes 1 min\n", + " !rapidfireai init --evals # Takes 1 min" + ] + }, + { + "cell_type": "markdown", + "id": "rSnMUj4iDc8Z", + "metadata": { + "id": "rSnMUj4iDc8Z" + }, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "Fm4CuRX5MzhZ", + "metadata": { + "id": "Fm4CuRX5MzhZ" + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "3b703a70", + "metadata": { + "id": "3b703a70" + }, + "source": [ + "### Import RapidFire Components" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3f8598e1", + "metadata": { + "id": "3f8598e1" + }, + "outputs": [], + "source": [ + "import os\n", + "os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python'\n", + "\n", + "from rapidfireai import Experiment\n", + "from rapidfireai.automl import List, RFLangChainRagSpec, RFvLLMModelConfig, RFPromptManager, RFGridSearch\n", + "import re, json\n", + "from typing import List as listtype, Dict, Any\n", + "\n", + "# NB: If you get \"AttributeError: 'MessageFactory' object has no attribute 'GetPrototype'\" from Colab, just rerun this cell" + ] + }, + { + "cell_type": "markdown", + "id": "9e16327b", + "metadata": { + "id": "9e16327b" + }, + "source": [ + "### Load Dataset, Rename Columns, and Downsample Data\n", + "\n", + "sample_fraction set extremely low (0.1) to avoid Colab disconnect" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee571098", + "metadata": { + "id": "ee571098" + }, + "outputs": [], + "source": [ + "from datasets import load_dataset\n", + "import pandas as pd\n", + "import json, random\n", + "from pathlib import Path\n", + "\n", + "#FiQA dataset\n", + "dataset_dir = Path(\"/content/tutorial_notebooks/rag-contexteng/datasets\")\n", + "\n", + "#Load all the files\n", + "fiqa_dataset = load_dataset(\"json\", data_files=str(dataset_dir / \"fiqa\" / \"queries.jsonl\"), split=\"train\")\n", + "fiqa_dataset = fiqa_dataset.rename_columns({\"text\": \"query\", \"_id\": \"query_id\"})\n", + "qrels = pd.read_csv(str(dataset_dir / \"fiqa\" / \"qrels.tsv\"), sep=\"\\t\")\n", + "qrels = qrels.rename(\n", + " columns={\"query-id\": \"query_id\", \"corpus-id\": \"corpus_id\", \"score\": \"relevance\"}\n", + ")\n", + "\n", + "#Downsample queries and corpus JOINTLY\n", + "sample_fraction = 0.1\n", + "rseed = 1\n", + "random.seed(rseed)\n", + "\n", + "# Sample queries\n", + "sample_size = int(len(fiqa_dataset) * sample_fraction)\n", + "fiqa_dataset = fiqa_dataset.shuffle(seed=rseed).select(range(sample_size))\n", + "\n", + "# Convert query_ids to integers for matching\n", + "query_ids = set([int(qid) for qid in fiqa_dataset[\"query_id\"]])\n", + "\n", + "# All the corpus docs should now be pointing to a relevant query\n", + "qrels_filtered = qrels[qrels[\"query_id\"].isin(query_ids)]\n", + "relevant_corpus_ids = set(qrels_filtered[\"corpus_id\"].tolist())\n", + "\n", + "print(f\"Using {len(fiqa_dataset)} queries\")\n", + "print(f\"Found {len(relevant_corpus_ids)} relevant documents for these queries\")\n", + "\n", + "# Load corpus and filter to relevant docs\n", + "input_file = dataset_dir / \"fiqa\" / \"corpus.jsonl\"\n", + "output_file = dataset_dir / \"fiqa\" / \"corpus_sampled.jsonl\"\n", + "\n", + "with open(input_file, 'r') as f:\n", + " all_corpus = [json.loads(line) for line in f]\n", + "\n", + "# Filter out any irrelevant documents\n", + "sampled_corpus = [doc for doc in all_corpus if int(doc[\"_id\"]) in relevant_corpus_ids]\n", + "\n", + "# Write sampled corpus\n", + "with open(output_file, 'w') as f:\n", + " for doc in sampled_corpus:\n", + " f.write(json.dumps(doc) + '\\n')\n", + "\n", + "print(f\"Sampled {len(sampled_corpus)} documents from {len(all_corpus)} total\")\n", + "print(f\"Saved to: {output_file}\")\n", + "print(f\"Filtered qrels to {len(qrels_filtered)} relevance judgments\")\n", + "\n", + "# Update qrels to match\n", + "qrels = qrels_filtered" + ] + }, + { + "cell_type": "markdown", + "id": "28399289", + "metadata": { + "id": "28399289" + }, + "source": [ + "### Create Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "70816920", + "metadata": { + "id": "70816920" + }, + "outputs": [], + "source": [ + "experiment = Experiment(experiment_name=\"exp1-fiqa-rag-colab\", mode=\"evals\")" + ] + }, + { + "cell_type": "markdown", + "id": "a73a21ee", + "metadata": { + "id": "a73a21ee" + }, + "source": [ + "### Define Partial Multi-Config Knobs for LangChain part of RAG Pipeline using RapidFire AI Wrapper APIs\n", + "\n", + "Note: encoding algorithm here is gpt2 with chunk size 150 and overlap 20 as well as chunk size 200 with overlao 60" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02b73586", + "metadata": { + "id": "02b73586" + }, + "outputs": [], + "source": [ + "from langchain_community.document_loaders import DirectoryLoader, JSONLoader\n", + "from langchain_text_splitters import RecursiveCharacterTextSplitter\n", + "from langchain_huggingface import HuggingFaceEmbeddings\n", + "from langchain_classic.retrievers.document_compressors import CrossEncoderReranker\n", + "from langchain_community.cross_encoders import HuggingFaceCrossEncoder\n", + "\n", + "# Per-Actor batch size for hardware efficiency\n", + "batch_size = 50\n", + "\n", + "\n", + "rag_gpu = RFLangChainRagSpec(\n", + " document_loader=DirectoryLoader(\n", + " path=str(dataset_dir / \"fiqa\"),\n", + " glob=\"corpus_sampled.jsonl\",\n", + " loader_cls=JSONLoader,\n", + " loader_kwargs={\n", + " \"jq_schema\": \".\",\n", + " \"content_key\": \"text\",\n", + " \"metadata_func\": lambda record, metadata: {\n", + " \"corpus_id\": int(record.get(\"_id\"))\n", + " }, # store the document id\n", + " \"json_lines\": True,\n", + " \"text_content\": False,\n", + " },\n", + " sample_seed=42,\n", + " ),\n", + " # chunking strategies with different chunk sizes (data chunking knob varied)\n", + " text_splitter=List([\n", + " RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n", + " encoding_name=\"gpt2\", chunk_size=150, chunk_overlap=20\n", + " ),\n", + " RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n", + " encoding_name=\"gpt2\", chunk_size=200, chunk_overlap=60\n", + " ),\n", + " ],\n", + " ),\n", + " embedding_cls=HuggingFaceEmbeddings,\n", + " embedding_kwargs={\n", + " \"model_name\": \"sentence-transformers/all-MiniLM-L6-v2\",\n", + " \"model_kwargs\": {\"device\": \"cuda:0\"},\n", + " \"encode_kwargs\": {\"normalize_embeddings\": True, \"batch_size\": batch_size},\n", + " },\n", + " vector_store=None, # uses FAISS by default\n", + " search_type=\"similarity\",\n", + " search_kwargs={\"k\": 2},\n", + " # 2 reranking strategies with different top-n values (reranking knob varied)\n", + " reranker_cls=CrossEncoderReranker,\n", + " reranker_kwargs={\n", + " \"model_name\": \"cross-encoder/ms-marco-MiniLM-L6-v2\",\n", + " \"model_kwargs\": {\"device\": \"cpu\"},\n", + " \"top_n\": List([1, 2]),\n", + " },\n", + " enable_gpu_search=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "dd6fb0a8", + "metadata": { + "id": "dd6fb0a8" + }, + "source": [ + "### Define Data Processing and Postprocessing Functions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ecc17276", + "metadata": { + "id": "ecc17276" + }, + "outputs": [], + "source": [ + "def sample_preprocess_fn(\n", + " batch: Dict[str, listtype], rag: RFLangChainRagSpec, prompt_manager: RFPromptManager\n", + ") -> Dict[str, listtype]:\n", + " \"\"\"Function to prepare the final inputs given to the generator model\"\"\"\n", + "\n", + " INSTRUCTIONS = \"Utilize your financial knowledge, give your answer or opinion to the input question or subject matter.\"\n", + "\n", + " # Perform batched retrieval over all queries; returns a list of lists of k documents per query\n", + " all_context = rag.get_context(batch_queries=batch[\"query\"], serialize=False)\n", + "\n", + "\n", + " retrieved_documents = [\n", + " [doc.metadata[\"corpus_id\"] for doc in docs] for docs in all_context\n", + " ]\n", + "\n", + "\n", + " serialized_context = rag.serialize_documents(all_context)\n", + " batch[\"query_id\"] = [int(query_id) for query_id in batch[\"query_id\"]]\n", + " return {\n", + " \"prompts\": [\n", + " [\n", + " {\"role\": \"system\", \"content\": INSTRUCTIONS},\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": f\"Here is some relevant context:\\n{context}. \\nNow answer the following question using the context provided earlier:\\n{question}\",\n", + " },\n", + " ]\n", + " for question, context in zip(batch[\"query\"], serialized_context)\n", + " ],\n", + " \"retrieved_documents\": retrieved_documents,\n", + " **batch,\n", + " }\n", + "\n", + "\n", + "def sample_postprocess_fn(batch: Dict[str, listtype]) -> Dict[str, listtype]:\n", + " \"\"\"Function to postprocess outputs produced by generator model\"\"\"\n", + " batch[\"ground_truth_documents\"] = [\n", + " qrels[qrels[\"query_id\"] == query_id][\"corpus_id\"].tolist()\n", + " for query_id in batch[\"query_id\"]\n", + " ]\n", + " return batch" + ] + }, + { + "cell_type": "markdown", + "id": "39eb16c3", + "metadata": { + "id": "39eb16c3" + }, + "source": [ + "### Define Custom Eval Metrics Functions\n", + "Note: MRR is the focus of the experiment which meausres if the #1 ranked value is the correct retrieved document" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22773d53", + "metadata": { + "id": "22773d53" + }, + "outputs": [], + "source": [ + "import math\n", + "\n", + "\n", + "def compute_ndcg_at_k(retrieved_docs: set, expected_docs: set, k=5):\n", + " \"\"\"Utility function to compute NDCG@k\"\"\"\n", + " relevance = [1 if doc in expected_docs else 0 for doc in list(retrieved_docs)[:k]]\n", + " dcg = sum(rel / math.log2(i + 2) for i, rel in enumerate(relevance))\n", + "\n", + " # IDCG: perfect ranking limited by min(k, len(expected_docs))\n", + " ideal_length = min(k, len(expected_docs))\n", + " ideal_relevance = [3] * ideal_length + [0] * (k - ideal_length)\n", + " idcg = sum(rel / math.log2(i + 2) for i, rel in enumerate(ideal_relevance))\n", + "\n", + " return dcg / idcg if idcg > 0 else 0.0\n", + "\n", + "\n", + "def compute_rr(retrieved_docs: set, expected_docs: set):\n", + " \"\"\"Utility function to compute Reciprocal Rank (RR) for a single query\"\"\"\n", + " rr = 0\n", + " for i, retrieved_doc in enumerate(retrieved_docs):\n", + " if retrieved_doc in expected_docs:\n", + " rr = 1 / (i + 1)\n", + " break\n", + " return rr\n", + "\n", + "\n", + "def sample_compute_metrics_fn(batch: Dict[str, listtype]) -> Dict[str, Dict[str, Any]]:\n", + " \"\"\"Function to compute all eval metrics based on retrievals and/or generations\"\"\"\n", + "\n", + " true_positives, precisions, recalls, f1_scores, ndcgs, rrs = 0, [], [], [], [], []\n", + " total_queries = len(batch[\"query\"])\n", + "\n", + " for pred, gt in zip(batch[\"retrieved_documents\"], batch[\"ground_truth_documents\"]):\n", + " expected_set = set(gt)\n", + " retrieved_set = set(pred)\n", + "\n", + " true_positives = len(expected_set.intersection(retrieved_set))\n", + " precision = true_positives / len(retrieved_set) if len(retrieved_set) > 0 else 0\n", + " recall = true_positives / len(expected_set) if len(expected_set) > 0 else 0\n", + " f1 = (\n", + " 2 * precision * recall / (precision + recall)\n", + " if (precision + recall) > 0\n", + " else 0\n", + " )\n", + "\n", + " precisions.append(precision)\n", + " recalls.append(recall)\n", + " f1_scores.append(f1)\n", + " ndcgs.append(compute_ndcg_at_k(retrieved_set, expected_set, k=5))\n", + " rrs.append(compute_rr(retrieved_set, expected_set))\n", + "## below will return the correct metrics\n", + " return {\n", + " \"Total\": {\"value\": total_queries},\n", + " \"Precision\": {\"value\": sum(precisions) / total_queries},\n", + " \"Recall\": {\"value\": sum(recalls) / total_queries},\n", + " \"F1 Score\": {\"value\": sum(f1_scores) / total_queries},\n", + " \"NDCG@5\": {\"value\": sum(ndcgs) / total_queries},\n", + " \"MRR\": {\"value\": sum(rrs) / total_queries},\n", + " }\n", + "\n", + "\n", + "def sample_accumulate_metrics_fn(\n", + " aggregated_metrics: Dict[str, listtype],\n", + ") -> Dict[str, Dict[str, Any]]:\n", + " \"\"\"Function to accumulate eval metrics across all batches\"\"\"\n", + "\n", + " num_queries_per_batch = [m[\"value\"] for m in aggregated_metrics[\"Total\"]]\n", + " total_queries = sum(num_queries_per_batch)\n", + " algebraic_metrics = [\"Precision\", \"Recall\", \"F1 Score\", \"NDCG@5\", \"MRR\"]\n", + "\n", + " return {\n", + " \"Total\": {\"value\": total_queries},\n", + " **{\n", + " metric: {\n", + " \"value\": sum(\n", + " m[\"value\"] * queries\n", + " for m, queries in zip(\n", + " aggregated_metrics[metric], num_queries_per_batch\n", + " )\n", + " )\n", + " / total_queries,\n", + " \"is_algebraic\": True,\n", + " \"value_range\": (0, 1),\n", + " }\n", + " for metric in algebraic_metrics\n", + " },\n", + " }" + ] + }, + { + "cell_type": "markdown", + "id": "c57887bc", + "metadata": { + "id": "c57887bc" + }, + "source": [ + "### Define Partial Multi-Config Knobs for vLLM Generator part of RAG Pipeline using RapidFire AI Wrapper APIs\n", + "\n", + " Qwen2.5-0.5B-Instruct (0.5B parameters) is perfect for Colab's memory constraints and feasible for this experiment\n", + "\n", + " Here also has the configs below which can be varied like max_model_len, use_fpc" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8f5d0824", + "metadata": { + "id": "8f5d0824" + }, + "outputs": [], + "source": [ + "vllm_config1 = RFvLLMModelConfig(\n", + " model_config={\n", + " \"model\": \"Qwen/Qwen2.5-0.5B-Instruct\",\n", + " \"dtype\": \"half\",\n", + " \"gpu_memory_utilization\": 0.25,\n", + " \"tensor_parallel_size\": 1,\n", + " \"distributed_executor_backend\": \"mp\",\n", + " \"enable_chunked_prefill\": False,\n", + " \"enable_prefix_caching\": False,\n", + " \"max_model_len\": 6000,\n", + " \"disable_log_stats\": True, # Disable vLLM progress logging\n", + " \"enforce_eager\": True,\n", + " \"disable_custom_all_reduce\": True,\n", + " },\n", + " sampling_params={\n", + " \"temperature\": 0.8,\n", + " \"top_p\": 0.95,\n", + " \"max_tokens\": 128,\n", + " },\n", + " rag=rag_gpu,\n", + " prompt_manager=None,\n", + ")\n", + "\n", + "batch_size = 3 # Smaller batch size for generation\n", + "config_set = {\n", + " \"vllm_config\": vllm_config1, # Only 1 generator, but it represents 4 full configs\n", + " \"batch_size\": batch_size,\n", + " \"preprocess_fn\": sample_preprocess_fn,\n", + " \"postprocess_fn\": sample_postprocess_fn,\n", + " \"compute_metrics_fn\": sample_compute_metrics_fn,\n", + " \"accumulate_metrics_fn\": sample_accumulate_metrics_fn,\n", + " \"online_strategy_kwargs\": {\n", + " \"strategy_name\": \"normal\",\n", + " \"confidence_level\": 0.95,\n", + " \"use_fpc\": True,\n", + " },\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "3a7dd280", + "metadata": { + "id": "3a7dd280" + }, + "source": [ + "### Create Config Group" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "67f26d9d", + "metadata": { + "id": "67f26d9d" + }, + "outputs": [], + "source": [ + "config_group = RFGridSearch(config_set)" + ] + }, + { + "cell_type": "markdown", + "id": "6e8a0e92", + "metadata": { + "id": "6e8a0e92" + }, + "source": [ + "### Display Ray Dashboard" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c7447480", + "metadata": { + "id": "c7447480" + }, + "outputs": [], + "source": [ + "from google.colab import output\n", + "output.serve_kernel_port_as_iframe(8855)" + ] + }, + { + "cell_type": "markdown", + "id": "fa186134", + "metadata": { + "id": "fa186134" + }, + "source": [ + "### Run Multi-Config Evals + Launch Interactive Run Controller\n", + "\n", + "\n", + "RapidFire AI also provides an Interactive Controller panel UI for Colab that lets you manage executing runs dynamically in real-time from the notebook:\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e07274a", + "metadata": { + "collapsed": true, + "id": "8e07274a" + }, + "outputs": [], + "source": [ + "# Launch evals of all RAG configs in the config_group with swap granularity of 4 chunks\n", + "results = experiment.run_evals(\n", + " config_group=config_group,\n", + " dataset=fiqa_dataset,\n", + " num_actors=1,\n", + " num_shards=4,\n", + " seed=42,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "r285brW5GKvE", + "metadata": { + "id": "r285brW5GKvE" + }, + "source": [ + "Here results_df returns the data frame that gives the correct metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "127e7b4a", + "metadata": { + "collapsed": true, + "id": "127e7b4a" + }, + "outputs": [], + "source": [ + "# Convert results dict to DataFrame\n", + "results_df = pd.DataFrame([\n", + " {k: v['value'] if isinstance(v, dict) and 'value' in v else v for k, v in {**metrics_dict, 'run_id': run_id}.items()}\n", + " for run_id, (_, metrics_dict) in results.items()\n", + "])\n", + "\n", + "results_df" + ] + }, + { + "cell_type": "markdown", + "id": "83pRy7DPGQ8l", + "metadata": { + "id": "83pRy7DPGQ8l" + }, + "source": [ + "This code checks to see if the results printed out the metrics you like" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "vZbq5jB-5ft8", + "metadata": { + "id": "vZbq5jB-5ft8" + }, + "outputs": [], + "source": [ + "print(results_df)" + ] + }, + { + "cell_type": "markdown", + "id": "to6OWeD8GePP", + "metadata": { + "id": "to6OWeD8GePP" + }, + "source": [ + "This adds plots to the folder in colab that can be viewed" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "w9SViGisMxPJ", + "metadata": { + "id": "w9SViGisMxPJ" + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "for metric in [\"Precision\", \"Recall\", \"F1 Score\", \"NDCG@5\", \"MRR\"]:\n", + " plt.figure()\n", + " plt.plot(results_df[\"run_id\"], results_df[metric], marker=\"o\")\n", + " plt.title(f\"{metric} vs run\")\n", + " plt.xlabel(\"Run ID\")\n", + " plt.ylabel(metric.upper())\n", + " plt.grid(True)\n", + " fname = f\"{metric}_plot.png\"\n", + " plt.savefig(fname)\n", + " plt.close()\n" + ] + }, + { + "cell_type": "markdown", + "id": "9135d951", + "metadata": { + "id": "9135d951" + }, + "source": [ + "### End Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "94ab038d", + "metadata": { + "id": "94ab038d" + }, + "outputs": [], + "source": [ + "from google.colab import output\n", + "from IPython.display import display, HTML\n", + "\n", + "display(HTML('''\n", + "\n", + "'''))\n", + "\n", + "# eval_js blocks until the Promise resolves\n", + "output.eval_js('''\n", + "new Promise((resolve) => {\n", + " document.getElementById(\"continue-btn\").onclick = () => {\n", + " document.getElementById(\"continue-btn\").disabled = true;\n", + " document.getElementById(\"continue-btn\").innerText = \"Continuing...\";\n", + " resolve(\"clicked\");\n", + " };\n", + "})\n", + "''')\n", + "\n", + "# Actually end the experiment after the button is clicked\n", + "experiment.end()\n", + "print(\"Done!\")" + ] + }, + { + "cell_type": "markdown", + "id": "09265e66", + "metadata": { + "id": "09265e66" + }, + "source": [ + "### View RapidFire AI Log Files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "05379a93", + "metadata": { + "id": "05379a93" + }, + "outputs": [], + "source": [ + "# Get the experiment-specific log file\n", + "log_file = experiment.get_log_file_path()\n", + "\n", + "print(f\"📄 Log File: {log_file}\")\n", + "print()\n", + "\n", + "if log_file.exists():\n", + " print(\"=\" * 80)\n", + " print(f\"Last 30 lines of {log_file.name}:\")\n", + " print(\"=\" * 80)\n", + " with open(log_file, 'r', encoding='utf-8') as f:\n", + " lines = f.readlines()\n", + " for line in lines[-30:]:\n", + " print(line.rstrip())\n", + "else:\n", + " print(f\"❌ Log file not found: {log_file}\")" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [], + "runtime_attributes": { + "runtime_version": "2025.10" + } + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}