import gensim
import MeCab
import pickle
from gensim.models.wrappers.fasttext import FastText
#import fasttext as ft
import random
import mojimoji
import numpy as np
from tqdm import tqdm

def ymyi(lis):
    wakati = MeCab.Tagger("-Owakati -b 81920 -u /clwork/ando/SEGBOT/MANBYO_201907_Dic-utf8.dic")

    with open('fm_space.pickle', 'rb') as f:
        fm = pickle.load(f)
    #model = gensim.models.KeyedVectors.load_word2vec_format("/clwork/ando/SEGBOT/cc.ja.300.vec", binary=False)
    model = ft.load_model("/clwork/ando/SEGBOT/fast/cc.ja.300.bin")
    texts = []
    sent = ""
    sparate = []
    label = []
    ruiseki = 0
    ruiseki2 = 0
    alls = []
    labels, text, num = [], [], []
    for n, line in enumerate(open(lis)):
        line = line.strip("\t").rstrip("\n")
        #print(line)
        if line == "":
            if sent == "":
                continue
            sent = wakati.parse(sent).split(" ")[:-1]
            flag = 0
            for i in sent:
                for j in sparate:
                    if ruiseki+len(i) > j and ruiseki < j:
                        label.append(1)
                        flag = 1
                    elif ruiseki+len(i) == j:
                        label.append(1)
                        flag = 1
                if flag == 0:
                    label.append(0)
                flag = 0
                ruiseki += len(i)
                #texts += i + " "
                try:
                    texts.append(model[i])
                    #texts.append(np.array(fm.vocab[i]))
                    #texts += str(fm.vocab[i].index) + " "
                    #print(i,str(fm.vocab[i].index))
                except KeyError:
                    texts.append(fm["<unk>"])
            label[-1] = 1
            #texts = texts.rstrip() + "\t"
            #texts += " ".join(label) + "\n"
            #alls.append((n,texts,label))
            labels.append(label)
            text.append(texts)
            num.append(n)
            sent = ""
            sparate = []
            texts = []
            label = []
            ruiseki = 0
            ruiseki2 = 0
            continue
        sent += mojimoji.han_to_zen(line, digit=False, ascii=False)
        ruiseki2 += len(line)
        sparate.append(ruiseki2)
    return num,text,labels

def nmni(lis):
    #wakati = MeCab.Tagger("-Owakati -b 81920 -u /clwork/ando/SEGBOT/MANBYO_201907_Dic-utf8.dic")
    wakati = MeCab.Tagger("-Owakati -b 81920")

    with open('fm_space.pickle', 'rb') as f:
        fm = pickle.load(f)
    #model = gensim.models.KeyedVectors.load_word2vec_format("/clwork/ando/SEGBOT/cc.ja.300.vec", binary=False)
    #model = ft.load_model("/clwork/ando/SEGBOT/fast/cc.ja.300.bin")
    texts = []
    sent = ""
    sparate = []
    label = []
    ruiseki = 0
    ruiseki2 = 0
    alls = []
    labels, text, num = [], [], []
    for n, line in enumerate(open(lis)):
        line = line.strip("\t").rstrip("\n")
        #print(line)
        if line == "":
            if sent == "":
                continue
            sent = wakati.parse(sent).split(" ")[:-1]
            flag = 0
            for i in sent:
                for j in sparate:
                    if ruiseki+len(i) > j and ruiseki < j:
                        label.append(1)
                        flag = 1
                    elif ruiseki+len(i) == j:
                        label.append(1)
                        flag = 1
                if flag == 0:
                    label.append(0)
                flag = 0
                ruiseki += len(i)
                #texts += i + " "
                try:
                    #texts.append(model[i])
                    texts.append(fm[i])
                    #texts += str(fm.vocab[i].index) + " "
                    #print(i,str(fm.vocab[i].index))
                except KeyError:
                    texts.append(fm["<unk>"])
            label[-1] = 1
            #texts = texts.rstrip() + "\t"
            #texts += " ".join(label) + "\n"
            #alls.append((n,texts,label))
            labels.append(label)
            text.append(texts)
            num.append(n)
            sent = ""
            sparate = []
            texts = []
            label = []
            ruiseki = 0
            ruiseki2 = 0
            continue
        sent += mojimoji.han_to_zen(line, digit=False, ascii=False)
        ruiseki2 += len(line)
        sparate.append(ruiseki2)
    return num,text,labels

def nmni_finetune(lis):
    #wakati = MeCab.Tagger("-Owakati -b 81920 -u /clwork/ando/SEGBOT/MANBYO_201907_Dic-utf8.dic")
    wakati = MeCab.Tagger("-Owakati -b 81920")
    #fm = gensim.models.KeyedVectors.load_word2vec_format('/clwork/ando/SEGBOT/cc.ja.300.vec', binary=False)
    with open('fm.pickle', 'rb') as f:
        fm = pickle.load(f)
    #fm = gensim.models.KeyedVectors.load_word2vec_format('cc.ja.300.vec', binary=False)
    #with open('fm.pickle', 'wb') as f:
    #    pickle.dump(fm, f)
    #model = gensim.models.KeyedVectors.load_word2vec_format("/clwork/ando/SEGBOT/cc.ja.300.vec", binary=False)
    #model = ft.load_model("/clwork/ando/SEGBOT/fast/cc.ja.300.bin")
    texts = []
    sent = ""
    sparate = []
    label = []
    ruiseki = 0
    ruiseki2 = 0
    alls = []
    labels, text, num = [], [], []
    for n, line in enumerate(open(lis)):
        line = line.strip("\t").rstrip("\n")
        #print(line)
        if line == "":
            if sent == "":
                continue
            sent = wakati.parse(sent).split(" ")[:-1]
            flag = 0
            for i in sent:
                for j in sparate:
                    if ruiseki+len(i) > j and ruiseki < j:
                        label.append(1)
                        flag = 1
                    elif ruiseki+len(i) == j:
                        label.append(1)
                        flag = 1
                if flag == 0:
                    label.append(0)
                flag = 0
                ruiseki += len(i)
                #texts += i + " "
                try:
                    #texts.append(model[i])
                    #texts.append(fm[i])
                    texts.append(fm.vocab[i].index)
                    #print(i,str(fm.vocab[i].index))
                except KeyError:
                    texts.append(fm.vocab["<unk>"].index)
            label[-1] = 1
            #texts = texts.rstrip() + "\t"
            #texts += " ".join(label) + "\n"
            #alls.append((n,texts,label))
            labels.append(np.array(label))
            text.append(np.array(texts))
            num.append(n)
            sent = ""
            sparate = []
            texts = []
            label = []
            ruiseki = 0
            ruiseki2 = 0
            continue
        sent += mojimoji.han_to_zen(line, digit=False, ascii=False)
        ruiseki2 += len(line)
        sparate.append(ruiseki2)
    return text,labels



def nmni_carte(lis):
    #wakati = MeCab.Tagger("-Owakati -b 81920 -u /clwork/ando/SEGBOT/MANBYO_201907_Dic-utf8.dic")
    wakati = MeCab.Tagger("-Owakati -b 81920")
    #fm = gensim.models.KeyedVectors.load_word2vec_format('/clwork/ando/SEGBOT/cc.ja.300.vec', binary=False)
    #fm = gensim.models.KeyedVectors.load_word2vec_format('cc.ja.300.vec', binary=False)
    #with open('fm.pickle', 'wb') as f:
    #    pickle.dump(fm, f)
    #model = gensim.models.KeyedVectors.load_word2vec_format("/clwork/ando/SEGBOT/cc.ja.300.vec", binary=False)
    #model = ft.load_model("/clwork/ando/SEGBOT/fast/cc.ja.300.bin")
    with open('fm.pickle', 'rb') as f:
        fm = pickle.load(f)
    texts = []
    sent = ""
    sparate = []
    label = []
    ruiseki = 0
    ruiseki2 = 0
    alls = []
    labels, text, num = [], [], []
    allab, altex, fukugenss = [], [], []
    #for n in tqdm(range(26431)):
    for n in tqdm(range(108)):
        fukugens = []
        for line in open(lis+str(n)+".txt"):
            line = line.strip()
            if line == "":
                continue
            sent = wakati.parse(line).split(" ")[:-1]
            flag = 0
            label = []
            texts = []
            fukugen = []
            for i in sent:
                try:
                    texts.append(fm.vocab[i].index)
                except KeyError:
                    texts.append(fm.vocab["<unk>"].index)
                fukugen.append(i)
                label.append(0)
            label[-1] = 1
            labels.append(np.array(label))
            text.append(np.array(texts))
            #labels.append(label)
            #text.append(texts)
            fukugens.append(fukugen)
        allab.append(labels)
        altex.append(text)
        fukugenss.append(fukugens)
        labels, text, fukugens= [], [], []
    return altex, allab, fukugenss


def nmni_finetune_s(lis):
    #wakati = MeCab.Tagger("-Owakati -b 81920 -u /clwork/ando/SEGBOT/MANBYO_201907_Dic-utf8.dic")
    wakati = MeCab.Tagger("-Owakati -b 81920")
    #fm = gensim.models.KeyedVectors.load_word2vec_format('/clwork/ando/SEGBOT/cc.ja.300.vec', binary=False)
    fm = gensim.models.KeyedVectors.load_word2vec_format('cc.ja.300.vec', binary=False)
    with open('fm.pickle', 'wb') as f:
        pickle.dump(fm, f)
    #model = gensim.models.KeyedVectors.load_word2vec_format("/clwork/ando/SEGBOT/cc.ja.300.vec", binary=False)
    #model = ft.load_model("/clwork/ando/SEGBOT/fast/cc.ja.300.bin")
    texts = []
    sent = ""
    sparate = []
    label = []
    ruiseki = 0
    ruiseki2 = 0
    alls = []
    labels, text, num = [], [], []
    for n, line in enumerate(open(lis)):
        line = line.strip("\t").rstrip("\n")
        sent = wakati.parse(line).split(" ")[:-1]
        flag = 0
        label = []
        texts = []
        for i in sent:
            try:
                texts.append(fm.vocab[i].index)
            except KeyError:
                texts.append(fm.vocab["<unk>"].index)
            label.append(0)
        label[-1] = 1
        labels.append(np.array(label))
        text.append(np.array(texts))
    return text,labels


def nmni_finetune_ss(lis):
    #wakati = MeCab.Tagger("-Owakati -b 81920 -u /clwork/ando/SEGBOT/MANBYO_201907_Dic-utf8.dic")
    wakati = MeCab.Tagger("-Owakati -b 81920")
    fm = gensim.models.KeyedVectors.load_word2vec_format('cc.ja.300.vec', binary=False)
    with open('fm.pickle', 'wb') as f:
        pickle.dump(fm, f)
    #with open('fm.pickle', 'rb') as f:
    #    fm = pickle.load(f)
    #model = gensim.models.KeyedVectors.load_word2vec_format("/clwork/ando/SEGBOT/cc.ja.300.vec", binary=False)
    #model = ft.load_model("/clwork/ando/SEGBOT/fast/cc.ja.300.bin")
    t,l =[],[]
    for i in range(108):
        texts = []
        sent = ""
        sparate = []
        label = []
        ruiseki = 0
        ruiseki2 = 0
        alls = []
        labels, text, num = [], [], []
        for n, line in enumerate(open(lis+str(i)+".txt")):
            line = line.strip("\t").rstrip("\n")
            if line == "":
                continue
            sent = wakati.parse(line).split(" ")[:-1]
            flag = 0
            label = []
            texts = []
            for i in sent:
                try:
                    texts.append(fm.vocab[i].index)
                except KeyError:
                    texts.append(fm.vocab["<unk>"].index)
                label.append(0)
            label[-1] = 1
            labels.append(np.array(label))
            text.append(np.array(texts))
        t.append(text)
        l.append(labels)
    return t,l

#model = ft.load_model("/clwork/ando/SEGBOT/fast/cc.ja.300.bin")
#print(model.get_subwords("間質性肺炎"))
#print(model.get_subwords("誤嚥性肺炎"))
#print(model.get_subwords("談話ユニット分割"))

"""
texts = []
sent = ""
sparate = []
label = []
ruiseki = 0
ruiseki2 = 0
alls = []
for n, line in enumerate(open("/clwork/ando/SEGBOT/randomdata.tsv")):
    line = line.strip("\t").rstrip("\n")
    if line == "":
        if sent == "":
            continue
        alls.append(sent)
        sent = ""
        continue
    else:
        sent += line    
if len(sent) != 0:
    alls.append(sent)
random.shuffle(alls)
#v = random.sample(alls, 300)
#for i in v:
#    alls.remove(i)
#t = random.sample(alls, 300)
#for i in t:
#    alls.remove(i)
with open("randomdata_concat.tsv","a")as f:
    f.write("\n".join())
#with open("dev_fix.tsv","a")as f:
#    for i in v:
#        f.write("\n".join(i))
#        f.write("\n\n")
#with open("test_fix.tsv","a")as f:
#    for i in t:
#        f.write("\n".join(i))
#        f.write("\n\n")
"""

"""
out = ""
for line in open("/clwork/ando/SEGBOT_BERT/alldata2_bert.tsv"):
    line = line.split("\t")
    line = line[0].strip()
    if line == "" or "サマリ" in line:
        continue
    out += line + "\n"
with open("alldata3.tsv","w")as f:
    f.write(out)
"""
"""
#wakati = MeCab.Tagger("-Owakati -b 81920 -u /clwork/ando/SEGBOT/MANBYO_201907_Dic-utf8.dic")
wakati = MeCab.Tagger("-Owakati -b 81920")

with open('fm_space.pickle', 'rb') as f:
    fm = pickle.load(f)
#model = gensim.models.KeyedVectors.load_word2vec_format("/clwork/ando/SEGBOT/cc.ja.300.vec", binary=False)
#model = ft.load_model("/clwork/ando/SEGBOT/fast/cc.ja.300.bin")
texts = []
sent = ""
sparate = []
label = []
ruiseki = 0
ruiseki2 = 0
alls = []
for n, line in enumerate(open("/clwork/ando/SEGBOT/train_fix.tsv")):
    line = line.strip("\t").rstrip("\n")
    #print(line)
    if line == "":
        if sent == "":
            continue
        sent = wakati.parse(sent).split(" ")[:-1]
        flag = 0
        for i in sent:
            for j in sparate:
                if ruiseki+len(i) > j and ruiseki < j:
                    label.append(1)
                    flag = 1
                elif ruiseki+len(i) == j:
                    label.append(1)
                    flag = 1
            if flag == 0:
                label.append(0)
            flag = 0
            ruiseki += len(i)
            #texts += i + " "
            try:
                #texts.append(model[i])
                texts.append(fm.vocab[i])
                #texts += str(fm.vocab[i].index) + " "
                #print(i,str(fm.vocab[i].index))
            except KeyError:
                texts.append(fm.vocab["<unk>"])
                print(i)
        label[-1] = 1
        #texts = texts.rstrip() + "\t"
        #texts += " ".join(label) + "\n"
        alls.append((str(n),texts,label))
        sent = ""
        sparate = []
        texts = []
        label = []
        ruiseki = 0
        ruiseki2 = 0
        continue
    sent += mojimoji.han_to_zen(line, digit=False, ascii=False)
    ruiseki2 += len(line)
    sparate.append(ruiseki2)
with open('nm_ni/train.pickle', 'wb') as f:
            pickle.dump(alls, f)
#print(alls)
#with open("resepdata_seped.tsv","w")as f:
#    f.write(texts)
"""



wakati = MeCab.Tagger("-Owakati")

#fm = gensim.models.KeyedVectors.load_word2vec_format('/clwork/ando/SEGBOT/cc.ja.300.vec', binary=False)
#with open('fm.pickle', 'wb') as f:
#        pickle.dump(fm, f)
texts = ""
sent = ""
sparate = []
label = []
ruiseki = 0
ruiseki2 = 0
for line in open("alldata.tsv"):
    line = line.split("\t")
    line = line[0].strip()
    if line == "" or "サマリ" in line:
        if sent == "":
            continue
        sent = wakati.parse(sent).split(" ")[:-1]
        flag = 0
        #print(sent,sparate)
        for i in sent:
            #print(i)
            for j in sparate:
                if ruiseki+len(i) > j and ruiseki < j:
                    #print(j)
                    label.append("1")
                    flag = 1
                elif ruiseki+len(i) == j:
                    #print(j)
                    label.append("1")
                    flag = 1
            if flag == 0:
                label.append("0")
            flag = 0
            ruiseki += len(i)
            #texts += i + " "
            
            try:
                texts += str(0) + " "
            except KeyError:
                print(i)
                #texts += str(fm.vocab["<unk>"].index) + " "
            
        label[-1] = "1"
        texts = texts.rstrip() + "\t"
        texts += " ".join(label) + "\n"
        sent = ""
        sparate = []
        label = []
        ruiseki = 0
        ruiseki2 = 0
        #print(texts)
        continue
    sent += line.strip()
    ruiseki2 += len(line.strip())
    sparate.append(ruiseki2)
with open("random_labbeled.tsv","w")as f:
    f.write(texts)





"""
wakati = MeCab.Tagger("-Owakati -u /clwork/ando/SEGBOT/MANBYO_201907_Dic-utf8.dic")


#fm = gensim.models.KeyedVectors.load_word2vec_format('/clwork/ando/SEGBOT/cc.ja.300_space.vec', binary=False)
#with open('fm_space.pickle', 'wb') as f:
#        pickle.dump(fm, f)

with open('fm_space.pickle', 'rb') as f:
    fm = pickle.load(f)
texts = ""
sent = ""
sparate = []
label = []
ruiseki = 0
ruiseki2 = 0
for line in open("/clwork/ando/SEGBOT/alldata_resep.tsv"):
    line = line.split("\t")
    line = line[0].strip("\t").rstrip("\n")
    #print(line)
    if line == "" or "サマリ" in line:
        if sent == "":
            continue
        print(sent)
        sent = sent.replace(" ","<space>")
        sent = wakati.parse(sent).split(" ")[:-1]
        print(sent)
        flag = 0
        #print(sent,sparate)
        for i in sent:
            #print(i)
            for j in sparate:
                if ruiseki+len(i) > j and ruiseki < j:
                    #print(j)
                    label.append("1")
                    flag = 1
                elif ruiseki+len(i) == j:
                    #print(j)
                    label.append("1")
                    flag = 1
            if flag == 0:
                label.append("0")
            flag = 0
            ruiseki += len(i)
            #texts += i + " "
            
            try:
                texts += str(fm.vocab[i].index) + " "
                #print(i,str(fm.vocab[i].index))
            except KeyError:
                texts += str(fm.vocab["<unk>"].index) + " "
        label[-1] = "1"
        texts = texts.rstrip() + "\t"
        texts += " ".join(label) + "\n"
        sent = ""
        sparate = []
        label = []
        ruiseki = 0
        ruiseki2 = 0
        #print(texts)
        continue
    sent += line.strip("\t")
    ruiseki2 += len(line)
    sparate.append(ruiseki2)
with open("alldata2_space.tsv","w")as f:
    f.write(texts)
"""



"""
wakati = MeCab.Tagger("-Owakati")

fm = gensim.models.KeyedVectors.load_word2vec_format('/clwork/ando/SEGBOT/cc.ja.300.vec', binary=False)
texts = ""
sent = ""
cand = ""
sparate = []
label = []
ruiseki = 0
ruiseki2 = 0
flag2 = 1
for line in open("data2.tsv"):
    line = line.split("\t")
    if flag2 == 1:
        cand = line
        flag2 = 2
        continue
    if flag2 == 2:
        flag2 = 1
        #print(line,cand)
        for n,z in enumerate(zip(cand,line)):
            i = z[0]
            j = z[1]
            n = n+1
            if i == "":
                sent = wakati.parse(sent).split(" ")[:-1]
                flag = 0
                #print(sent,sparate)
                for i in sent:
                    #print(i)
                    for j in sparate:
                        if ruiseki+len(i) > j and ruiseki < j:
                            #print(j)
                            label.append("1")
                            flag = 1
                        elif ruiseki+len(i) == j:
                            #print(j)
                            label.append("1")
                            flag = 1
                    if flag == 0:
                        label.append("0")
                    flag = 0
                    ruiseki += len(i)
                    #texts += i + " "

                    try:
                        texts += str(fm.vocab[i].index) + " "
                    except KeyError:
                        texts += str(fm.vocab["<unk>"].index) + " "

                label[-1] = "1"
                texts = texts.rstrip() + "\t"
                texts += " ".join(label) + "\n"
                sent = ""
                sparate = []
                label = []
                ruiseki = 0
                ruiseki2 = 0
                #print(texts)
                break
            if j == "|":
                sparate.append(n)
            sent += i
with open("alldata.tsv","w")as f:
    f.write(texts)
"""