Spaces:
Runtime error
Runtime error
| # -*- coding: utf-8 -*- | |
| """ | |
| Author: Philipp Seidl | |
| ELLIS Unit Linz, LIT AI Lab, Institute for Machine Learning | |
| Johannes Kepler University Linz | |
| Contact: [email protected] | |
| Plot utils | |
| """ | |
| import numpy as np | |
| import pandas as pd | |
| import matplotlib.pyplot as plt | |
| from matplotlib import pyplot as plt | |
| plt.style.use('default') | |
| def normal_approx_interval(p_hat, n, z=1.96): | |
| """ approximating the distribution of error about a binomially-distributed observation, {\hat {p)), with a normal distribution | |
| z = 1.96 --> alpha =0.05 | |
| z = 1 --> std | |
| https://www.wikiwand.com/en/Binomial_proportion_confidence_interval""" | |
| return z*((p_hat*(1-p_hat))/n)**(1/2) | |
| our_colors = { | |
| "lightblue": ( 0/255, 132/255, 187/255), | |
| "red": (217/255, 92/255, 76/255), | |
| "blue": ( 0/255, 132/255, 187/255), | |
| "green": ( 91/255, 167/255, 85/255), | |
| "yellow": (241/255, 188/255, 63/255), | |
| "cyan": ( 79/255, 176/255, 191/255), | |
| "grey": (125/255, 130/255, 140/255), | |
| "lightgreen":(191/255, 206/255, 82/255), | |
| "violett": (174/255, 97/255, 157/255), | |
| } | |
| def plot_std(p_hats, n_samples,z=1.96, color=our_colors['red'], alpha=0.2, xs=None): | |
| p_hats = np.array(p_hats) | |
| stds = np.array([normal_approx_interval(p_hats[ii], n_samples[ii], z=z) for ii in range(len(p_hats))]) | |
| xs = range(len(p_hats)) if xs is None else xs | |
| plt.fill_between(xs, p_hats-(stds), p_hats+stds, color=color, alpha=alpha) | |
| #plt.errorbar(range(13), asdf, [normal_approx_interval(asdf[ii], n_samples[ii], z=z) for ii in range(len(asdf))], | |
| # c=our_colors['red'], linestyle='None', marker='.', ecolor=our_colors['red']) | |
| def plot_loss(hist): | |
| plt.plot(hist['step'], hist['loss'] ) | |
| plt.plot(hist['steps_valid'], np.array(hist['loss_valid'])) | |
| plt.legend(['train','validation']) | |
| plt.xlabel('update-step') | |
| plt.ylabel('loss (categorical-crossentropy-loss)') | |
| def plot_topk(hist, sets=['train', 'valid', 'test'], with_last = 2): | |
| ks = [1, 2, 3, 4, 5, 10, 20, 30, 40, 50, 100] | |
| baseline_val_res = {1:0.4061, 10:0.6827, 50: 0.7883, 100:0.8400} | |
| plt.plot(list(baseline_val_res.keys()), list(baseline_val_res.values()), 'k.--') | |
| for i in range(1,with_last): | |
| for s in sets: | |
| plt.plot(ks, [hist[f't{k}_acc_{s}'][-i] for k in ks],'.--', alpha=1/i) | |
| plt.xlabel('top-k') | |
| plt.ylabel('Accuracy') | |
| plt.legend(sets) | |
| plt.title('Hopfield-NN') | |
| plt.ylim([-0.02,1]) | |
| def plot_nte(hist, dataset='Sm', last_cpt=1, include_bar=True, model_legend='MHN (ours)', | |
| draw_std=True, z=1.96, n_samples=None, group_by_template_fp=False, schwaller_hist=None, fortunato_hist=None): #1.96 for 95%CI | |
| markers = ['.']*4#['1','2','3','4']#['8','P','p','*'] | |
| lw = 2 | |
| ms = 8 | |
| k = 100 | |
| ntes = range(13) | |
| if dataset=='Sm': | |
| basel_values = [0. , 0.38424785, 0.66807858, 0.7916149 , 0.9051132 , | |
| 0.92531258, 0.87295875, 0.94865587, 0.91830721, 0.95993717, | |
| 0.97215858, 0.9896713 , 0.99917817] #old basel_values = [0.0, 0.3882, 0.674, 0.7925, 0.9023, 0.9272, 0.874, 0.947, 0.9185, 0.959, 0.9717, 0.9927, 1.0] | |
| pretr_values = [0.08439423, 0.70743412, 0.85555528, 0.95200267, 0.96513376, | |
| 0.96976397, 0.98373613, 0.99960286, 0.98683919, 0.96684724, | |
| 0.95907246, 0.9839079 , 0.98683919]# old [0.094, 0.711, 0.8584, 0.952, 0.9683, 0.9717, 0.988, 1.0, 1.0, 0.984, 0.9717, 1.0, 1.0] | |
| staticQK = [0.2096, 0.1992, 0.2291, 0.1787, 0.2301, 0.1753, 0.2142, 0.2693, 0.2651, 0.1786, 0.2834, 0.5366, 0.6636] | |
| if group_by_template_fp: | |
| staticQK = [0.2651, 0.2617, 0.261 , 0.2181, 0.2622, 0.2393, 0.2157, 0.2184, 0.2 , 0.225 , 0.2039, 0.4568, 0.5293] | |
| if dataset=='Lg': | |
| pretr_values = [0.03410448, 0.65397054, 0.7254572 , 0.78969294, 0.81329924, | |
| 0.8651173 , 0.86775655, 0.8593128 , 0.88184124, 0.87764794, | |
| 0.89734215, 0.93328846, 0.99531597] | |
| basel_values = [0. , 0.62478044, 0.68784314, 0.75089511, 0.77044644, | |
| 0.81229423, 0.82968149, 0.82965544, 0.83778338, 0.83049176, | |
| 0.8662873 , 0.92308414, 1.00042408] | |
| #staticQK = [0.03638, 0.0339 , 0.03732, 0.03506, 0.03717, 0.0331 , 0.03003, 0.03613, 0.0304 , 0.02109, 0.0297 , 0.02632, 0.02217] # on 90k templates | |
| staticQK = [0.006416,0.00686, 0.00616, 0.00825, 0.005085,0.006718,0.01041, 0.0015335,0.006668,0.004673,0.001706,0.02551,0.04074] | |
| if dataset=='Golden': | |
| staticQK = [0]*13 | |
| pretr_values = [0]*13 | |
| basel_values = [0]*13 | |
| if schwaller_hist: | |
| midx = np.argmin(schwaller_hist['loss_valid']) | |
| basel_values = ([schwaller_hist[f't100_acc_nte_{k}'][midx] for k in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, '>10', '>49']]) | |
| if fortunato_hist: | |
| midx = np.argmin(fortunato_hist['loss_valid']) | |
| pretr_values = ([fortunato_hist[f't100_acc_nte_{k}'][midx] for k in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, '>10', '>49']]) | |
| #hand_val = [0.0 , 0.4, 0.68, 0.79, 0.89, 0.91, 0.86, 0.9,0.88, 0.9, 0.93] | |
| if include_bar: | |
| if dataset=='Sm': | |
| if n_samples is None: | |
| n_samples = [610, 1699, 287, 180, 143, 105, 70, 48, 124, 86, 68, 2539, 1648] | |
| if group_by_template_fp: | |
| n_samples = [460, 993, 433, 243, 183, 117, 102, 87, 110, 80, 103, 3048, 2203] | |
| if dataset=='Lg': | |
| if n_samples is None: | |
| n_samples = [18861, 32226, 4220, 2546, 1573, 1191, 865, 652, 1350, 642, 586, 11638, 4958] #new | |
| if group_by_template_fp: | |
| n_samples = [13923, 17709, 7637, 4322, 2936, 2137, 1586, 1260, 1272, 1044, 829, 21695, 10559] | |
| #[5169, 15904, 2814, 1853, 1238, 966, 766, 609, 1316, 664, 640, 30699, 21471] | |
| #[13424,17246, 7681, 4332, 2844,2129,1698,1269, 1336,1067, 833, 22491, 11202] #grouped fp | |
| plt.bar(range(11+2), np.array(n_samples)/sum(n_samples[:-1]), alpha=0.4, color=our_colors['grey']) | |
| xti = [*[str(i) for i in range(11)], '>10', '>49'] | |
| asdf = [] | |
| for nte in xti: | |
| try: | |
| asdf.append( hist[f't{k}_acc_nte_{nte}'][-last_cpt]) | |
| except: | |
| asdf.append(None) | |
| plt.plot(range(13), asdf,f'{markers[3]}--', markersize=ms,c=our_colors['red'], linewidth=lw,alpha=1) | |
| plt.plot(ntes, pretr_values,f'{markers[1]}--', c=our_colors['green'], | |
| linewidth=lw, alpha=1,markersize=ms) #old [0.08, 0.7, 0.85, 0.9, 0.91, 0.95, 0.98, 0.97,0.98, 1, 1] | |
| plt.plot(ntes, basel_values,f'{markers[0]}--',linewidth=lw, | |
| c=our_colors['blue'], markersize=ms,alpha=1) | |
| plt.plot(range(len(staticQK)), staticQK, f'{markers[2]}--',markersize=ms,c=our_colors['yellow'],linewidth=lw, alpha=1) | |
| plt.title(f'USPTO-{dataset}') | |
| plt.xlabel('number of training examples') | |
| plt.ylabel('top-100 test-accuracy') | |
| plt.legend([model_legend, 'Fortunato et al.','FNN baseline',"FPM baseline", #static${\\xi X}: \\dfrac{|{\\xi} \\cap {X}|}{|{X}|}$ | |
| 'test sample proportion']) | |
| if draw_std: | |
| alpha=0.2 | |
| plot_std(asdf, n_samples, z=z, color=our_colors['red'], alpha=alpha) | |
| plot_std(pretr_values, n_samples, z=z, color=our_colors['green'], alpha=alpha) | |
| plot_std(basel_values, n_samples, z=z, color=our_colors['blue'], alpha=alpha) | |
| plot_std(staticQK, n_samples, z=z, color=our_colors['yellow'], alpha=alpha) | |
| plt.xticks(range(13),xti); | |
| plt.yticks(np.arange(0,1.05,0.1)) | |
| plt.grid('on', alpha=0.3) |