Hey @lefromage , there's an issue with the qwen attention implementation with MPS device on macbooks. CPU should work. Are you using it on a macbook with MPS by any chance?

does not make a difference , same issue with cpu and mps

device = 'cpu'

device = 'mps'

if 'v1' in model_id:
model = MxbaiRerankV1(model_id,
device=device,
)
else:
model = MxbaiRerankV2(model_id,
device=device,
)

This works for me. Please check your torch/transformers installation!

from mxbai_rerank import MxbaiRerankV2

reranker = MxbaiRerankV2(device="cpu")

query = '''which animal is purple'''
documents = [
'''rat is red''',
'''cat is blue''',
'''elephant is green''',
'''dog is yellow''',
'''giraffe is purple''',
]

reranker.rank(query, documents)

[RankResult(index=4, score=8.171875, document='giraffe is purple'),
 RankResult(index=1, score=2.5234375, document='cat is blue'),
 RankResult(index=0, score=2.203125, document='rat is red'),
 RankResult(index=3, score=1.75, document='dog is yellow'),
 RankResult(index=2, score=1.7109375, document='elephant is green')]

this only works with the default base model :
from mxbai_rerank import MxbaiRerankV2

query = '''which animal is purple'''
documents = [
'''rat is red''',
'''cat is blue''',
'''elephant is green''',
'''dog is yellow''',
'''giraffe is purple''',
]

for model in ['mixedbread-ai/mxbai-rerank-base-v2', 'mixedbread-ai/mxbai-rerank-large-v2']:
print(model)
reranker = MxbaiRerankV2(model, device="cpu")

results = reranker.rank(query, documents)
for result in results:
    print(result)

mixedbread-ai/mxbai-rerank-base-v2

RankResult(index=4, score=8.171875, document='giraffe is purple')
RankResult(index=1, score=2.5234375, document='cat is blue')
RankResult(index=0, score=2.1953125, document='rat is red')
RankResult(index=3, score=1.7578125, document='dog is yellow')

RankResult(index=2, score=1.71875, document='elephant is green')

mixedbread-ai/mxbai-rerank-large-v2

RankResult(index=0, score=nan, document='rat is red')
RankResult(index=1, score=nan, document='cat is blue')
RankResult(index=2, score=nan, document='elephant is green')
RankResult(index=3, score=nan, document='dog is yellow')
RankResult(index=4, score=nan, document='giraffe is purple')