|
|
|
import spacy |
|
from fast_sentence_tokenize import tokenize_text |
|
|
|
|
|
def evaluate_response(original_tokenized: str, response: str) -> int: |
|
""" |
|
- Tokenize response string using spacy |
|
- Create a list of response tokens |
|
- Assign every original token a rank: |
|
- Only look at the last mention of a token in the response |
|
- Rank the tokens by how early they appear in the response (last mention only) |
|
- Calculate ranking accuracy |
|
|
|
Returns a value between 0 and 1 |
|
""" |
|
original_tokenized = original_tokenized.strip().lower() |
|
response = response.strip().lower() |
|
|
|
|
|
response_tokens = tokenize_text(response) |
|
|
|
|
|
original_tokens = original_tokenized.split() |
|
|
|
|
|
response_token_ranks = {} |
|
for token in original_tokens: |
|
if token not in response_tokens: |
|
return 0 |
|
|
|
|
|
original_token_ranks = {} |
|
for i, token in enumerate(original_tokens): |
|
original_token_ranks[token] = i |
|
|
|
|
|
for token in original_tokens: |
|
|
|
response_token_ranks[token] = len(response_tokens) - 1 - response_tokens[::-1].index(token) |
|
|
|
|
|
sorted_ranks = sorted(set(response_token_ranks.values())) |
|
rank_mapping = {old_rank: new_rank for new_rank, old_rank in enumerate(sorted_ranks)} |
|
for token, rank in response_token_ranks.items(): |
|
response_token_ranks[token] = rank_mapping[rank] |
|
|
|
|
|
n = len(original_tokens) |
|
concordant_pairs = 0 |
|
discordant_pairs = 0 |
|
|
|
for i in range(n): |
|
for j in range(i + 1, n): |
|
original_diff = original_token_ranks[original_tokens[i]] - original_token_ranks[original_tokens[j]] |
|
response_diff = response_token_ranks[original_tokens[i]] - response_token_ranks[original_tokens[j]] |
|
|
|
if original_diff * response_diff > 0: |
|
concordant_pairs += 1 |
|
elif original_diff * response_diff < 0: |
|
discordant_pairs += 1 |
|
|
|
total_pairs = n * (n - 1) // 2 |
|
kendall_tau = (concordant_pairs - discordant_pairs) / total_pairs |
|
|
|
|
|
normalized_kendall_tau = (kendall_tau + 1) / 2 |
|
|
|
return normalized_kendall_tau |
|
|