File size: 7,588 Bytes
9a70c5d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 |
import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.losses import binary_crossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import load_model
from sklearn.metrics import roc_curve
from scipy.interpolate import interp1d
from scipy.optimize import brentq
import matplotlib.pyplot as plt
from scipy.io.wavfile import read
from sklearn.preprocessing import normalize
from generate_array_feature import mald_feature, get_filelist
import time
from pydub import AudioSegment
import whisper
import os
import spacy
# To deal with one wav file.
def is_command_reasonable(command, time, location):
commands = [
"OK Google.",
"Turn on Bluetooth.",
"Record a video.",
"Take a photo.",
"Open music player.",
"Set an alarm for 6:30 am.",
"Remind me to buy coffee at 7 am.",
"What is my schedule for tomorrow?",
"Square root of 2105?",
"Open browser.",
"Decrease volume.",
"Turn on flashlight.",
"Set the volume to full.",
"Mute the volume.",
"What's the definition of transmit?",
"Call Pizza Hut.",
"Call the nearest computer shop.",
"Show me my messages.",
"Translate please give me directions to Chinese.",
"How do you say good night in Japanese?"
]
# Time : Work-0 / Rest-1 / Sleep-2
# Location : Work-0 / Home-1
commands_daily = [
"Call Pizza Hut.",
"Remind me to buy coffee at 7 am.",
"Open music player.",
"Record a video.",
"Take a photo.",
]
commands_work = [
"Open browser.",
"What is my schedule for tomorrow?",
"Square root of 2105?",
"Call the nearest computer shop.",
"Show me my messages.",
"Translate please give me directions to Chinese.",
"How do you say good night in Japanese?",
"What's the definition of transmit?",
]
commands_basic = [
"OK Google.",
"Decrease volume.",
"Turn on Bluetooth.",
"Turn on flashlight.",
"Set the volume to full.",
"Mute the volume.",
"Set an alarm for 6:30 am."]
if time == 0 and location == 0:
if command in commands_daily:
return False
else:
return True
elif time ==2:
if command in commands_basic:
return True
else:
return False
else:
if command in commands_work:
return False
else:
return True
def convert_6ch_wav_to_stereo(input_file_path, output_file_path):
sound = AudioSegment.from_file(input_file_path, format="wav")
if sound.channels != 6:
raise ValueError("The input file does not have 6 channels.")
front_left = sound.split_to_mono()[0]
front_right = sound.split_to_mono()[1]
center = sound.split_to_mono()[2]
back_left = sound.split_to_mono()[4]
back_right = sound.split_to_mono()[5]
center = center - 6
back_left = back_left - 6
back_right = back_right - 6
stereo_left = front_left.overlay(center).overlay(back_left)
stereo_right = front_right.overlay(center).overlay(back_right)
stereo_sound = AudioSegment.from_mono_audiosegments(stereo_left, stereo_right)
stereo_sound.export(output_file_path, format="wav")
def judge_human(rate,data):
model = load_model('/home/fazhong/Github/czx/data-task0_1/train1.keras')
feature =list(mald_feature(rate, data))
features=np.array([feature])
y_pred = model.predict(features)
return y_pred[0]
def judge_name(rate,data):
model = load_model('/home/fazhong/Github/czx/data-task0/train1.keras')
feature =list(mald_feature(rate, data))
features=np.array([feature])
y_pred = model.predict(features)
y_pred_classes = np.argmax(y_pred,axis=1)
return y_pred_classes[0]
def judge_command(file_path):
whisper_model = whisper.load_model("large")
out_path='/home/fazhong/Github/czx/temp/temp.wav'
convert_6ch_wav_to_stereo(file_path,out_path)
# print(out_path)
result = whisper_model.transcribe(out_path,language="en")
text_result = result['text']
print(text_result)
return text_result
def judge_classifier(command):
nlp = spacy.load('en_core_web_md')
commands = [
"OK Google.",
"Turn on Bluetooth.",
"Record a video.",
"Take a photo.",
"Open music player.",
"Set an alarm for 6:30 am.",
"Remind me to buy coffee at 7 am.",
"What is my schedule for tomorrow?",
"Square root of 2105?",
"Open browser.",
"Decrease volume.",
"Turn on flashlight.",
"Set the volume to full.",
"Mute the volume.",
"What’s the definition of transmit?",
"Call Pizza Hut.",
"Call the nearest computer shop.",
"Show me my messages.",
"Translate please give me directions to Chinese.",
"How do you say good night in Japanese?"
]
def classify_key(command):
if 'ok google' in command:
return 1
elif 'bluetooth' in command and 'on' in command:
return 2
elif 'record' in command and 'video' in command:
return 3
elif 'take' in command and 'photo' in command:
return 4
elif 'music player' in command and 'open' in command:
return 5
elif 'set' in command and 'alarm' in command:
return 6
elif 'remind' in command and 'coffee' in command:
return 7
elif 'schedule' in command or 'tomorrow' in command:
return 8
elif 'square root' in command:
return 9
elif 'open browser' in command:
return 10
elif 'decrease volume' in command:
return 11
elif 'flashlight' in command and 'on' in command:
return 12
elif 'volume' in command and 'full' in command:
return 13
elif 'mute' in command and 'volume' in command:
return 14
elif 'definition of' in command:
return 15
elif 'call' in command and 'pizza hut' in command.lower():
return 16
elif 'call' in command and 'computer shop' in command.lower():
return 17
elif 'messages' in command and 'show' in command:
return 18
elif 'translate' in command:
return 19
elif 'good night' in command and 'in japanese' in command:
return 20
else:
return None # or some default value if command is not recognized
file_content = command
result_pre = classify_key(file_content.replace('.', '').replace(',', '').lower().strip())
if result_pre is not None:
return result_pre
input_doc = nlp(file_content.replace('.', '').replace(',', '').lower().strip())
similarities = [(command, input_doc.similarity(nlp(command))) for command in commands]
best_match = max(similarities, key=lambda item: item[1])
return best_match[0]
def judge(file_path,time,location):
rate, data = read(file_path)
# Maybe change to paths?
temp = judge_human(rate,data)
temp2 = judge_name(rate,data)
command = judge_command(file_path)
text = judge_classifier(command)
if is_command_reasonable(text, time, location):
return True
else:
return False
if __name__ == "__main__":
judge('/home/fazhong/Github/czx2/example/data/fengattack60/feng_attack_echo_60_01_3.150-4.000.wav',0,0) |