CP/sim_score_plot.py at main · stevolopolis/CP · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
import matplotlib.pyplot as plt
from scipy.stats import pearsonr

from utils import *

def plot_analytical_score_loss_corr_combined(analytical_score, loss_score, save_path=None):
    """Scatter plot of similarity score vs loss score"""
    corr = pearsonr(analytical_score, loss_score)

    plt.scatter(analytical_score, loss_score)
    plt.xlabel('Similarity score')
    plt.ylabel('Loss score')
    plt.title('Similarity score vs Loss score\n(corr: %0.2f, prob null: %0.2f)' % (corr[0], corr[1]))

    plt.ylim(0, max(loss_score) * 1.25)

    for i in range(len(analytical_score)):
        plt.annotate(i, (analytical_score[i], loss_score[i]))

    if save_path is not None:
        plt.savefig(save_path)
        print("Saved to %s" % save_path)
    else:
        plt.show()

    plt.close()


def plot_corr_vs_n_pieces(n_pieces_ls, corr_ls, save_path=None):
    """Line plot of correlation vs n_pieces"""
    plt.plot(n_pieces_ls, corr_ls, label=['Correlation', 'NULL Proc'])
    plt.xlabel('n_pieces')
    plt.ylabel('Correlation')
    plt.title('Correlation vs n_pieces')
    plt.legend()

    if save_path is not None:
        plt.savefig(save_path)
        print("Saved to %s" % save_path)
    else:
        plt.show()

    plt.close()


def load_val(path):
    with open(path, 'r') as f:
        val = float(f.read().strip())
        # check if val is infs or nans
        if val == float('inf') or val == float('-inf') or val != val:
            print(f"Value at {path} is {val}")
            return -1
        return val


if __name__ == '__main__':
    n_trials = 70
    n_pieces = 30
    dynamic = True
    MODEL = 'relu'

    corr_save_path = f"vis/toy_signal/correlation"
    create_subdirectories(corr_save_path)

    # ===============================================
    # Correlation between analytical score and loss
    # ===============================================
    # rdp = []
    # rdpAnal = []
    # loss = []
    # for trial in range(n_trials):
    #     model_path = f"results/{MODEL}/{MODEL}_{trial}"
    #     analytical_save_path = "vis/toy_signal/analytical/%s" % trial
    #     empirical_save_path = "vis/toy_signal/empirical/%s" % trial

    #     rdp_score = load_val(f"{analytical_save_path}/rdp{n_pieces}_score.txt")
    #     rdpAnal_score = load_val(f"{analytical_save_path}/rdpAnal{n_pieces}_score.txt")
    #     model_loss = load_val(f"{empirical_save_path}/loss.txt")

    #     if rdp_score == -1 or rdpAnal_score == -1 or model_loss == -1:
    #         continue

    #     rdp.append(rdp_score)
    #     rdpAnal.append(rdpAnal_score)
    #     loss.append(model_loss)

    # plot_analytical_score_loss_corr_combined(rdp, loss, save_path=f"{corr_save_path}/rdp{n_pieces}_loss.png")
    # plot_analytical_score_loss_corr_combined(rdpAnal, loss, save_path=f"{corr_save_path}/rdpAnal{n_pieces}_loss.png")


    # # ===============================================
    # # Relationship between score-loss correlation and n_pieces
    # # ===============================================
    # rdp_corr_ls = []
    # rdpAnal_corr_ls = []
    # n_pieces_ls = [n for n in range(10, 101, 10)]
    # for n_pieces in n_pieces_ls:
    #     rdp = []
    #     rdpAnal = []
    #     ema = []
    #     loss = []
    #     for trial in range(n_trials):
    #         model_path = f"results/{MODEL}/{MODEL}_{trial}"
    #         analytical_save_path = "vis/toy_signal/analytical/%s" % trial
    #         empirical_save_path = "vis/toy_signal/empirical/%s" % trial

    #         rdp_score = load_val(f"{analytical_save_path}/rdp{n_pieces}_score.txt")
    #         rdpAnal_score = load_val(f"{analytical_save_path}/rdpAnal{n_pieces}_score.txt")
    #         model_loss = load_val(f"{empirical_save_path}/loss.txt")

    #         if rdp_score == -1 or rdpAnal_score == -1 or model_loss == -1:
    #             continue

    #         rdp.append(rdp_score)
    #         rdpAnal.append(rdpAnal_score)
    #         loss.append(model_loss)

    #     rdp_corr_ls.append(pearsonr(rdp, loss))
    #     rdpAnal_corr_ls.append(pearsonr(rdpAnal, loss))

    # plot_corr_vs_n_pieces(n_pieces_ls, rdp_corr_ls, save_path=f"{corr_save_path}/rdp_corr_vs_n_pieces.png")
    # plot_corr_vs_n_pieces(n_pieces_ls, rdpAnal_corr_ls, save_path=f"{corr_save_path}/rdpAnal_corr_vs_n_pieces.png")


    # ===============================================
    # Matched correlation between analytical score and loss
    # ===============================================
    rdp = []
    loss = []
    n_pieces_ls = [n for n in range(10, 101, 10)]
    for trial in range(n_trials):
        model_path = f"results/{MODEL}/{MODEL}_{trial}"
        analytical_save_path = "vis/toy_signal/analytical/%s" % trial
        empirical_save_path = "vis/toy_signal/empirical/%s" % trial

        model_loss = load_vals(f"{empirical_save_path}/loss.txt")[0]
        loss.append(model_loss)

        rdp_score_ls = []
        for n_pieces in n_pieces_ls:
            rdp_score = load_vals(f"{analytical_save_path}/rdp{n_pieces}_score.txt")[0]

            if rdp_score == -1 or model_loss == -1:
                continue

            rdp_score_ls.append(rdp_score)

        matching_idx = np.argmin(np.abs(np.array(rdp_score_ls) - model_loss))
        rdp.append(rdp_score_ls[matching_idx])

    plot_analytical_score_loss_corr_combined(rdp, loss, save_path=f"{corr_save_path}/rdp_matched_loss.png")