import numpy as np from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense, Dropout from tensorflow.keras.losses import binary_crossentropy from tensorflow.keras.optimizers import Adam from tensorflow.keras.models import load_model from tensorflow.keras.callbacks import ModelCheckpoint from tensorflow.keras.utils import to_categorical import tensorflow as tf from sklearn.metrics import roc_curve from scipy.interpolate import interp1d from scipy.optimize import brentq import os import random import spacy import matplotlib.pyplot as plt # == Part 1 - Read data == data = np.load("/home/fazhong/Github/czx/data.npy", allow_pickle=True) labels = np.load("/home/fazhong/Github/czx/labels.npy", allow_pickle=True) texts = np.load("/home/fazhong/Github/czx/texts.npy", allow_pickle=True) commands = [ "OK Google.", "Turn on Bluetooth.", "Record a video.", "Take a photo.", "Open music player.", "Set an alarm for 6:30 am.", "Remind me to buy coffee at 7 am.", "What is my schedule for tomorrow?", "Square root of 2105?", "Open browser.", "Decrease volume.", "Turn on flashlight.", "Set the volume to full.", "Mute the volume.", "What's the definition of transmit?", "Call Pizza Hut.", "Call the nearest computer shop.", "Show me my messages.", "Translate please give me directions to Chinese.", "How do you say good night in Japanese?" ] commands_basic = [ 0,# "OK Google.", 1,#"Turn on Bluetooth.", 5,#"Set an alarm for 6:30 am.", 10,#"Decrease volume.", 11,#"Turn on flashlight.", 12,#"Set the volume to full.", 13,#"Mute the volume.", ] commands_daily = [ 2,#"Record a video.", 3,#"Take a photo.", 4,#"Open music player.", 6,#"Remind me to buy coffee at 7 am.", 15,#"Call Pizza Hut.", ] commands_work = [ 7,#"What is my schedule for tomorrow?", 8,#"Square root of 2105?", 9,#"Open browser.", 14,#"What's the definition of transmit?", 16,#"Call the nearest computer shop.", 17,#"Show me my messages.", 18,#"Translate please give me directions to Chinese.", 19,#"How do you say good night in Japanese?" ] def rule_judge(type,time,location): if type in commands_basic: if time == 0: return False else: return True elif type in commands_daily: if time == 2: return True else: return False elif type in commands_work: if time == 1 and location ==1: return True else: return False # 0 - sleep time / 1 - work time / 2 - daily time times_label = [0,1,2] # 0 - home / 1 - factory location_label = [0,1] data_all = [] data = data.tolist() labels = labels.tolist() texts = texts.tolist() acc_num = 0 all_num = len(data) atk_list = [] atk_err = [] name_err = [] type_err = [] gt_label = [] pre_label = [] name_err_num = [0,0,0,0] name_acc_num = [0,0,0,0] command_err_num = [] command_acc_num = [] for i in range(20): command_err_num.append(0) command_acc_num.append(0) for i in range(len(data)): tmp = [] tmp.append(np.array(data[i][0])) tmp.extend([labels[i][0]]) tmp.extend([labels[i][1]]) tmp.extend([labels[i][2]]) data_all.append(tmp) data = data_all time_labels = [] location_labels = [] for i in range(len(data)): time_labels.append(random.randint(0,2)) location_labels.append(random.randint(0,1)) rule_err = [] for i in range(len(data)): if not rule_judge(data[i][2],time_labels[i],location_labels[i]): rule_err.append(i) # == Part 2 - Judge of Human == model = load_model('/home/fazhong/Github/czx/data-task0_1/train1.keras') X = np.asarray([x[0] for x in data]) y = np.asarray([x[1] for x in data]) type = np.asarray([x[3] for x in data]) y_pred = model.predict(X) y_pred = y_pred.reshape((len(y_pred), 1)) y = y.reshape((len(y), 1)) for i in range(len(y)): if(y_pred[i]>0.5):y_pred[i]=1 else: y_pred[i] = 0 atk_list.append(i) if(y_pred[i]!=y[i]): atk_err.append(i) ACCU = np.sum((y_pred == y)) / len(y) print(len(y)) print("ACCU is " + str(100 * ACCU)) # == Part 3 - Judge of Name == model = load_model('/home/fazhong/Github/czx/data-task0/train1.keras') y_name = np.asarray([x[2] for x in data]) y_pred = model.predict(X) y_pred_classes = np.argmax(y_pred,axis=1) ACCU = np.sum((y_pred_classes == y_name)) / len(y_name) for i in range(len(y_name)): if(y_pred_classes[i]!=y_name[i]): name_err.append(i) print("ACCU is " + str(100 * ACCU)) # Part 4 - Transcribe and Judge of Reason # PS! Attack的文本不需要跑分类 nlp = spacy.load('en_core_web_md') def classify_key(command): if 'ok google' in command: return 1 elif 'okay' in command: return 1 elif 'bluetooth' in command: return 2 elif 'record' in command and 'video' in command: return 3 elif 'take' in command and 'photo' in command: return 4 elif 'music' in command: return 5 elif 'alarm' in command: return 6 elif 'remind' in command and 'coffee' in command: return 7 elif 'am' in command : return 7 elif 'schedule' in command or 'tomorrow' in command: return 8 elif 'square root' in command: return 9 elif 'open browser' in command: return 10 elif 'decrease volume' in command: return 11 elif 'flashlight' in command and 'on' in command: return 12 elif 'hello freshlight' in command.lower(): return 12 elif 'turn on' in command: return 12 elif 'volume' in command and 'full' in command: return 13 elif 'mute' in command : return 14 elif 'move' in command : return 14 elif 'more' in command : return 14 elif 'motor' in command : return 14 elif 'mood' in command : return 14 elif 'most' in command : return 14 elif 'what' in command : return 14 elif 'with' in command : return 14 elif 'milk' in command : return 14 elif 'use' in command : return 14 elif 'definition of' in command: return 15 elif 'call' in command and 'pizza hut' in command.lower(): return 16 elif 'copies are' in command.lower() or 'call a piece of heart' in command.lower() or 'copies of' in command.lower(): return 16 elif 'peace' in command.lower(): return 16 elif 'heart' in command.lower(): return 16 elif 'pisa' in command.lower(): return 16 elif 'piece' in command.lower(): return 16 elif 'hard' in command.lower(): return 16 elif 'call' in command and 'computer shop' in command.lower(): return 17 elif 'message' in command : return 18 elif 'translate' in command: return 19 elif 'good night' in command and 'in japanese' in command: return 20 else: return None # or some default value if command is not recognized correct_count = 0 total_count = 0 category_number = 0 total_normal = 0 normal_texts = [] normal_labels = [] All_Normal_names = [] # Test of rule module test_flag = True atk_org_list = [] for i in range(len(texts)): if test_flag: normal_texts.append(texts[i]) All_Normal_names.append(y_name[i]) normal_labels.append(type[i]) if y[i] == 0: atk_org_list.append(i) else: if y[i] == 1: normal_texts.append(texts[i]) All_Normal_names.append(y_name[i]) normal_labels.append(type[i]) print(len(atk_org_list)) # for text in texts: # if texts.index(text) in atk_list: # print(texts.index(text)) # continue # else: # normal_texts.append(text) weird_name = [] weird_command = [] # for i in range(len(data)): # if not rule_judge(data[i][2],time_labels[i],location_labels[i]): # rule_err.append(i) for i in range(len(normal_texts)): text = normal_texts[i] category_number = normal_labels[i] # print(text) # print(category_number) result_pre = classify_key(text.replace('.', '').replace(',', '').lower().strip()) # IF rule - judge # if not rule_judge(category_number-1,time_labels[i],location_labels[i]): # command_err_num[category_number-1]+=1 # name_err_num[All_Normal_names[i]]+=1 # continue if i in atk_org_list: command_err_num[category_number-1]+=1 name_err_num[All_Normal_names[i]]+=1 continue if result_pre is not None: if result_pre == category_number: correct_count += 1 command_acc_num[category_number-1]+=1 name_acc_num[All_Normal_names[i]]+=1 continue input_doc = nlp(text.replace('.', '').replace(',', '').lower().strip()) similarities = [(command, input_doc.similarity(nlp(command))) for command in commands] best_match = max(similarities, key=lambda item: item[1]) best_match_index = commands.index(best_match[0]) + 1 if best_match_index == category_number: correct_count += 1 command_acc_num[category_number-1]+=1 name_acc_num[All_Normal_names[i]]+=1 else: # print(text.replace('.', '').replace(',', '').lower().strip()) # if category_number==16: # print(input_doc,commands[category_number-1],commands[best_match_index-1]) command_err_num[category_number-1]+=1 name_err_num[All_Normal_names[i]]+=1 # if 'thank' in str(input_doc): # pass # # print('?') # # print(texts.index(text)) # # print(data[texts.index(text)]) # weird_name.append(y_name[texts.index(text)]) # weird_command.append(type[texts.index(text)]) type_err.append(texts.index(text)) # 计算正确率 accuracy = correct_count / len(normal_texts) print(f"Accuracy: {accuracy:.2f}") # Part 5 - Results atk_set = set(atk_err) name_set = set(name_err) type_set = set(type_err) #rule_set = set(rule_err) err_list = list(atk_set | name_set | type_set) print(len(err_list)) # print(weird_name) print(name_err_num) print(name_acc_num) print(command_err_num) print(command_acc_num) # print(weird_command) #print(atk_list) # print(len(atk_list)) # print(all_num) # print(atk_err) # print(name_err) # print(type_err) # print(type_set) # print(err_list) # # 设置柱状图的位置编号 # x = np.arange(len(name_err_num)) # # 画柱状图 # plt.bar(x - 0.2, name_acc_num, width=0.4, label='Correct', color='green') # plt.bar(x + 0.2, name_err_num, width=0.4, label='Error', color='red') # # 添加标题和标签 # plt.xlabel('Names') # plt.ylabel('Counts') # plt.title('Accuracy and Errors by Name') # plt.xticks(x, ['User1', 'User2', 'User3', 'User4']) # 假设有四个名字 # plt.legend() # #plt.savefig('/home/fazhong/Github/czx/user.png') # # 显示图形 # plt.close() # # 设置柱状图的位置编号 # x = np.arange(len(command_err_num)) # # 画柱状图 # plt.bar(x - 0.2, command_acc_num, width=0.4, label='Correct', color='blue') # plt.bar(x + 0.2, command_err_num, width=0.4, label='Error', color='orange') # # 添加标题和标签 # plt.xlabel('Commands') # plt.ylabel('Counts') # plt.title('Accuracy and Errors by Command') # plt.xticks(x, [i for i in range(20)]) # 假设有六个命令 # plt.legend() # # 显示图形 # #plt.savefig('/home/fazhong/Github/czx/com.png')