tomaarsen's picture
tomaarsen HF Staff
Publish MIC usage, slim fixture and reviewed Transformers expectations
b61198d verified
|
Raw History Blame Contribute Delete
2.64 kB

SentenceTransformer: encoding and scoring

Run the public Sentence Transformers API with a small, fixed retrieval example.

import torch
from sentence_transformers import SentenceTransformer
import json  # doc-builder: hide
import os  # doc-builder: hide

model_id = 'nomic-ai/modernbert-embed-base'
model_id = os.environ.get('MIC_MODEL', model_id)  # doc-builder: hide
revision = os.environ.get('MIC_MODEL_REVISION') or None  # doc-builder: hide
variant = json.loads(os.environ.get('MIC_MODEL_KWARGS', '{}'))  # doc-builder: hide
torch.manual_seed(0)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
model = SentenceTransformer(
    model_id,
    device=device,
    revision=revision,  # doc-builder: hide
    model_kwargs=variant,  # doc-builder: hide
)
queries = ['Which planet is known as the Red Planet?', 'Where do penguins live?']
documents = ["Venus is often called Earth's twin.", 'Mars is known for its reddish appearance and is called the Red Planet.', 'Penguins live in the Southern Hemisphere, including Antarctica.', 'Jupiter is the largest planet in our solar system and has a prominent red spot.']
query_embeddings = model.encode_query(queries, show_progress_bar=False, convert_to_tensor=True)
document_embeddings = model.encode_document(documents, show_progress_bar=False, convert_to_tensor=True)
scores = model.similarity(query_embeddings, document_embeddings)
assert scores.shape == (2, 4)
assert torch.isfinite(scores).all()
assert torch.isfinite(query_embeddings).all() and torch.isfinite(document_embeddings).all()
assert (query_embeddings[0] - query_embeddings[1]).abs().max() > 0.001
individual = model.encode_query(queries, batch_size=1, show_progress_bar=False, convert_to_tensor=True)
batch_max_delta = float((query_embeddings - individual).abs().max())
if query_embeddings.dtype == torch.float32:
    torch.testing.assert_close(query_embeddings, individual, atol=1e-4, rtol=1e-4)

if os.environ.get('MIC_RUNNER'):  # doc-builder: hide
    backbone = next(module for module in model.modules() if isinstance(module, __import__('transformers').PreTrainedModel))
    result = {
        'runtime': {'device': model.device.type, 'dtype': str(next(model.parameters()).dtype).removeprefix('torch.'), 'attn_implementation': backbone.config._attn_implementation},
        'scores': scores.float().cpu().tolist(),
        'batch_max_delta': batch_max_delta,
        'query_embeddings': query_embeddings.float().cpu().tolist(),
        'document_embeddings': document_embeddings.float().cpu().tolist(),
    }
    print(json.dumps(result))