File size: 190 Bytes
4df8249
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
def clean(text):
    if text.endswith("</s>"):
        text = text[:-len("</s>")]

    if text.endswith("<|endoftext|>"):
        text = text[:-len("<|endoftext|>")]
        
    return text