Spaces:
Build error
Build error
| import _pickle as cPickle | |
| import multiprocessing | |
| import os | |
| import sys | |
| import numpy as np | |
| from joblib import Parallel, delayed | |
| import idnns.networks.network as nn | |
| from idnns.information import information_process as inn | |
| from idnns.plots import plot_figures as plt_fig | |
| from idnns.networks import network_paramters as netp | |
| from idnns.networks.utils import load_data | |
| # from idnns.network import utils | |
| # import idnns.plots.plot_gradients as plt_grads | |
| NUM_CORES = multiprocessing.cpu_count() | |
| class informationNetwork(): | |
| """A class that store the network, train it and calc it's information (can be several of networks) """ | |
| def __init__(self, rand_int=0, num_of_samples=None, args=None): | |
| if args == None: | |
| args = netp.get_default_parser(num_of_samples) | |
| self.cov_net = args.cov_net | |
| self.calc_information = args.calc_information | |
| self.run_in_parallel = args.run_in_parallel | |
| self.num_ephocs = args.num_ephocs | |
| self.learning_rate = args.learning_rate | |
| self.batch_size = args.batch_size | |
| self.activation_function = args.activation_function | |
| self.interval_accuracy_display = args.interval_accuracy_display | |
| self.save_grads = args.save_grads | |
| self.num_of_repeats = args.num_of_repeats | |
| self.calc_information_last = args.calc_information_last | |
| self.num_of_bins = args.num_of_bins | |
| self.interval_information_display = args.interval_information_display | |
| self.save_ws = args.save_ws | |
| self.name = args.data_dir + args.data_name | |
| # The arch of the networks | |
| self.layers_sizes = netp.select_network_arch(args.net_type) | |
| # The percents of the train data samples | |
| self.train_samples = np.linspace(1, 100, 199)[[[x * 2 - 2 for x in index] for index in args.inds]] | |
| # The indexs that we want to calculate the information for them in logspace interval | |
| self.epochs_indexes = np.unique( | |
| np.logspace(np.log2(args.start_samples), np.log2(args.num_ephocs), args.num_of_samples, dtype=int, | |
| base=2)) - 1 | |
| max_size = np.max([len(layers_size) for layers_size in self.layers_sizes]) | |
| # load data | |
| self.data_sets = load_data(self.name, args.random_labels) | |
| # create arrays for saving the data | |
| self.ws, self.grads, self.information, self.models, self.names, self.networks, self.weights = [ | |
| [[[[None] for k in range(len(self.train_samples))] for j in range(len(self.layers_sizes))] | |
| for i in range(self.num_of_repeats)] for _ in range(7)] | |
| self.loss_train, self.loss_test, self.test_error, self.train_error, self.l1_norms, self.l2_norms = \ | |
| [np.zeros((self.num_of_repeats, len(self.layers_sizes), len(self.train_samples), len(self.epochs_indexes))) | |
| for _ in range(6)] | |
| params = {'sampleLen': len(self.train_samples), | |
| 'nDistSmpls': args.nDistSmpls, | |
| 'layerSizes': ",".join(str(i) for i in self.layers_sizes[0]), 'nEpoch': args.num_ephocs, 'batch': args.batch_size, | |
| 'nRepeats': args.num_of_repeats, 'nEpochInds': len(self.epochs_indexes), | |
| 'LastEpochsInds': self.epochs_indexes[-1], 'DataName': args.data_name, | |
| 'lr': args.learning_rate} | |
| self.name_to_save = args.name + "_" + "_".join([str(i) + '=' + str(params[i]) for i in params]) | |
| params['train_samples'], params['CPUs'], params[ | |
| 'directory'], params['epochsInds'] = self.train_samples, NUM_CORES, self.name_to_save, self.epochs_indexes | |
| self.params = params | |
| self.rand_int = rand_int | |
| # If we trained already the network | |
| self.traind_network = False | |
| def save_data(self, parent_dir='jobs/', file_to_save='data.pickle'): | |
| """Save the data to the file """ | |
| directory = '{0}/{1}{2}/'.format(os.getcwd(), parent_dir, self.params['directory']) | |
| data = {'information': self.information, | |
| 'test_error': self.test_error, 'train_error': self.train_error, 'var_grad_val': self.grads, | |
| 'loss_test': self.loss_test, 'loss_train': self.loss_train, 'params': self.params | |
| , 'l1_norms': self.l1_norms, 'weights': self.weights, 'ws': self.ws} | |
| if not os.path.exists(directory): | |
| os.makedirs(directory) | |
| self.dir_saved = directory | |
| with open(self.dir_saved + file_to_save, 'wb') as f: | |
| cPickle.dump(data, f, protocol=2) | |
| def run_network(self): | |
| """Train and calculated the network's information""" | |
| if self.run_in_parallel: | |
| results = Parallel(n_jobs=NUM_CORES)(delayed(nn.train_network) | |
| (self.layers_sizes[j], | |
| self.num_ephocs, self.learning_rate, self.batch_size, | |
| self.epochs_indexes, self.save_grads, self.data_sets, | |
| self.activation_function, | |
| self.train_samples, self.interval_accuracy_display, | |
| self.calc_information, | |
| self.calc_information_last, self.num_of_bins, | |
| self.interval_information_display, self.save_ws, self.rand_int, | |
| self.cov_net) | |
| for i in range(len(self.train_samples)) for j in | |
| range(len(self.layers_sizes)) for k in range(self.num_of_repeats)) | |
| else: | |
| results = [nn.train_and_calc_inf_network(i, j, k, | |
| self.layers_sizes[j], | |
| self.num_ephocs, self.learning_rate, self.batch_size, | |
| self.epochs_indexes, self.save_grads, self.data_sets, | |
| self.activation_function, | |
| self.train_samples, self.interval_accuracy_display, | |
| self.calc_information, | |
| self.calc_information_last, self.num_of_bins, | |
| self.interval_information_display, | |
| self.save_ws, self.rand_int, self.cov_net) | |
| for i in range(len(self.train_samples)) for j in range(len(self.layers_sizes)) for k in | |
| range(self.num_of_repeats)] | |
| # Extract all the measures and orgainze it | |
| for i in range(len(self.train_samples)): | |
| for j in range(len(self.layers_sizes)): | |
| for k in range(self.num_of_repeats): | |
| index = i * len(self.layers_sizes) * self.num_of_repeats + j * self.num_of_repeats + k | |
| current_network = results[index] | |
| self.networks[k][j][i] = current_network | |
| self.ws[k][j][i] = current_network['ws'] | |
| self.weights[k][j][i] = current_network['weights'] | |
| self.information[k][j][i] = current_network['information'] | |
| self.grads[k][i][i] = current_network['gradients'] | |
| self.test_error[k, j, i, :] = current_network['test_prediction'] | |
| self.train_error[k, j, i, :] = current_network['train_prediction'] | |
| self.loss_test[k, j, i, :] = current_network['loss_test'] | |
| self.loss_train[k, j, i, :] = current_network['loss_train'] | |
| self.traind_network = True | |
| def print_information(self): | |
| """Print the networks params""" | |
| for val in self.params: | |
| if val != 'epochsInds': | |
| print (val, self.params[val]) | |
| def calc_information(self): | |
| """Calculate the infomration of the network for all the epochs - only valid if we save the activation values and trained the network""" | |
| if self.traind_network and self.save_ws: | |
| self.information = np.array( | |
| [inn.get_information(self.ws[k][j][i], self.data_sets.data, self.data_sets.labels, | |
| self.args.num_of_bins, self.args.interval_information_display, self.epochs_indexes) | |
| for i in range(len(self.train_samples)) for j in | |
| range(len(self.layers_sizes)) for k in range(self.args.num_of_repeats)]) | |
| else: | |
| print ('Cant calculate the infomration of the networks!!!') | |
| def calc_information_last(self): | |
| """Calculate the information of the last epoch""" | |
| if self.traind_network and self.save_ws: | |
| return np.array([inn.get_information([self.ws[k][j][i][-1]], self.data_sets.data, self.data_sets.labels, | |
| self.args.num_of_bins, self.args.interval_information_display, | |
| self.epochs_indexes) | |
| for i in range(len(self.train_samples)) for j in | |
| range(len(self.layers_sizes)) for k in range(self.args.num_of_repeats)]) | |
| def plot_network(self): | |
| str_names = [[self.dir_saved]] | |
| mode = 2 | |
| save_name = 'figure' | |
| plt_fig.plot_figures(str_names, mode, save_name) | |