diff --git a/wandb_data/performance.py b/wandb_data/performance.py index 8fcbead..35dadc9 100644 --- a/wandb_data/performance.py +++ b/wandb_data/performance.py @@ -12,7 +12,7 @@ def compute_miner_performance( download_fake_images=False, validator_run_name=None): - miner_perf = defaultdict(lambda: {'predictions': [], 'labels': []}) + miner_preds_labels = defaultdict(lambda: {'predictions': [], 'labels': []}) fake_image_preds = defaultdict(dict) for run in wandb_validator_runs: @@ -21,7 +21,7 @@ def compute_miner_performance( history_df = run.history() image_files = [f for f in run.files() if f.name.endswith(".png")] - for i, challenge_row in history_df.iterrows(): + for _, challenge_row in history_df.iterrows(): if start_ts is not None and challenge_row['_timestamp'] < start_ts: continue if end_ts is not None and challenge_row['_timestamp'] > end_ts: @@ -60,16 +60,20 @@ def compute_miner_performance( if pred == -1: continue - miner_perf[uid]['predictions'].append(pred) - miner_perf[uid]['labels'].append(label) + + miner_preds_labels[uid]['predictions'].append(pred) + miner_preds_labels[uid]['labels'].append(label) - metrics = {uid: compute_metrics(data['predictions'], data['labels']) for uid, data in miner_perf.items()} + metrics = { + uid: compute_metrics(data['predictions'], data['labels']) + for uid, data in miner_preds_labels.items() + } flattened_metrics = [] for uid, metric_dict in metrics.items(): flattened_metrics.append({'uid': uid, **metric_dict}) metrics_df = pd.DataFrame(flattened_metrics) - return metrics_df, fake_image_preds + return metrics_df, fake_image_preds, miner_preds_labels def compute_metrics(predictions, labels): diff --git a/wandb_data/wandb_miner_performance.ipynb b/wandb_data/wandb_miner_performance.ipynb index 213d449..52b150d 100644 --- a/wandb_data/wandb_miner_performance.ipynb +++ b/wandb_data/wandb_miner_performance.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 17, + "execution_count": 1, "id": "14848402081f829b", "metadata": { "ExecuteTime": { @@ -33,7 +33,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 2, "id": "cf87818a8afd8bb8", "metadata": { "ExecuteTime": { @@ -47,15 +47,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "Querying data for uid 0 starting at 2024-10-18T16:09:49\n" + "Querying data for uid 0 starting at 2024-10-22T13:09:49\n" ] } ], "source": [ "project = \"bitmind-subnet\" # mainnet wandb project name\n", "entity = \"bitmindai\"\n", - "validator_run_name = 'validator-122-1.2.1'\n", - "start_dt = '2024-10-18T16:09:49'\n", + "validator_run_name = None #'validator-122-1.2.2'\n", + "start_dt = '2024-10-22T13:09:49'\n", "# start_dt = formatted_ts_from_epoch(1729494162)\n", "end_dt = None\n", "\n", @@ -65,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 3, "id": "37dd8282", "metadata": {}, "outputs": [ @@ -73,7 +73,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Querying w&b with filters: {'display_name': 'validator-122-1.2.1', 'created_at': {'$gte': '2024-10-18T16:09:49'}}\n" + "Querying w&b with filters: {'created_at': {'$gte': '2024-10-22T13:09:49'}}\n" ] } ], @@ -83,7 +83,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 4, "id": "ae1fec8d", "metadata": {}, "outputs": [ @@ -121,20 +121,20 @@ " \n", " 0\n", " 0\n", - " 0.952703\n", - " 0.986014\n", - " 0.921569\n", - " 0.952703\n", - " 0.99223\n", - " 592\n", + " 0.852941\n", + " 1.0\n", + " 0.782609\n", + " 0.878049\n", + " 0.960474\n", + " 34\n", " \n", " \n", "\n", "" ], "text/plain": [ - " uid accuracy precision recall f1 auc sample_size\n", - "0 0 0.952703 0.986014 0.921569 0.952703 0.99223 592" + " uid accuracy precision recall f1 auc sample_size\n", + "0 0 0.852941 1.0 0.782609 0.878049 0.960474 34" ] }, "metadata": {}, @@ -142,7 +142,7 @@ } ], "source": [ - "metrics_df, fake_image_preds = compute_miner_performance(\n", + "metrics_df, fake_image_preds, miner_preds = compute_miner_performance(\n", " runs,\n", " miner_uid=miner_uid, # example uid\n", " start_ts=epoch_from_formatted_ts(start_dt),\n", @@ -285,7 +285,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 15, "id": "beff084a4853e438", "metadata": { "ExecuteTime": { @@ -299,15 +299,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "Querying data for uid None starting at 2024-10-19T16:09:49\n" + "Querying data for uid None starting at 2024-10-22T16:09:49\n" ] } ], "source": [ "project = \"bitmind\" # mainnet wandb project name\n", "entity = \"bitmindai\"\n", - "validator_run_name = 'validator-39-1.2.0'\n", - "start_dt = '2024-10-19T16:09:49'\n", + "validator_run_name = 'validator-39-1.2.2'\n", + "start_dt = '2024-10-22T16:09:49'\n", "# start_dt = formatted_ts_from_epoch(1729494162)\n", "end_dt = None\n", "\n", @@ -317,7 +317,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 16, "id": "d844f9310e28daa9", "metadata": { "ExecuteTime": { @@ -331,7 +331,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Querying w&b with filters: {'display_name': 'validator-39-1.2.0', 'created_at': {'$gte': '2024-10-19T16:09:49'}}\n" + "Querying w&b with filters: {'display_name': 'validator-39-1.2.2', 'created_at': {'$gte': '2024-10-22T16:09:49'}}\n" ] } ], @@ -341,7 +341,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 17, "id": "21ed23cf8d100746", "metadata": { "collapsed": false @@ -381,42 +381,62 @@ " \n", " 0\n", " 198\n", - " 0.525326\n", - " 0.505376\n", - " 0.982090\n", - " 0.667343\n", - " 0.605484\n", - " 691\n", + " 0.75\n", + " 0.75\n", + " 1.000000\n", + " 0.857143\n", + " 0.666667\n", + " 4\n", " \n", " \n", " 1\n", - " 196\n", - " 0.553687\n", - " 0.530410\n", - " 0.966495\n", - " 0.684932\n", - " 0.572205\n", - " 773\n", + " 218\n", + " 0.50\n", + " 1.00\n", + " 0.333333\n", + " 0.500000\n", + " 1.000000\n", + " 4\n", " \n", " \n", " 2\n", - " 130\n", - " 0.667162\n", - " 0.605568\n", - " 0.828571\n", - " 0.699732\n", - " 0.685439\n", - " 673\n", + " 225\n", + " 0.75\n", + " 0.75\n", + " 1.000000\n", + " 0.857143\n", + " 0.666667\n", + " 4\n", " \n", " \n", " 3\n", - " 125\n", - " 0.800718\n", - " 0.977273\n", - " 0.616487\n", - " 0.756044\n", - " 0.790426\n", - " 557\n", + " 217\n", + " 0.50\n", + " 1.00\n", + " 0.333333\n", + " 0.500000\n", + " 1.000000\n", + " 4\n", + " \n", + " \n", + " 4\n", + " 216\n", + " 0.75\n", + " 0.75\n", + " 1.000000\n", + " 0.857143\n", + " 0.666667\n", + " 4\n", + " \n", + " \n", + " 5\n", + " 196\n", + " 0.75\n", + " 0.75\n", + " 1.000000\n", + " 0.857143\n", + " 0.666667\n", + " 4\n", " \n", " \n", "\n", @@ -424,10 +444,12 @@ ], "text/plain": [ " uid accuracy precision recall f1 auc sample_size\n", - "0 198 0.525326 0.505376 0.982090 0.667343 0.605484 691\n", - "1 196 0.553687 0.530410 0.966495 0.684932 0.572205 773\n", - "2 130 0.667162 0.605568 0.828571 0.699732 0.685439 673\n", - "3 125 0.800718 0.977273 0.616487 0.756044 0.790426 557" + "0 198 0.75 0.75 1.000000 0.857143 0.666667 4\n", + "1 218 0.50 1.00 0.333333 0.500000 1.000000 4\n", + "2 225 0.75 0.75 1.000000 0.857143 0.666667 4\n", + "3 217 0.50 1.00 0.333333 0.500000 1.000000 4\n", + "4 216 0.75 0.75 1.000000 0.857143 0.666667 4\n", + "5 196 0.75 0.75 1.000000 0.857143 0.666667 4" ] }, "metadata": {}, @@ -435,7 +457,7 @@ } ], "source": [ - "metrics_df, fake_image_preds = compute_miner_performance(\n", + "metrics_df, fake_image_preds, miner_preds = compute_miner_performance(\n", " runs,\n", " miner_uid=miner_uid, # example uid\n", " start_ts=epoch_from_formatted_ts(start_dt),\n", @@ -445,6 +467,14 @@ "\n", "display(metrics_df)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a6e60aee", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {