config.json

{
  "architectures": [
    "OgmaModel"
  ],
  "model_type": "ogma",
  "auto_map": {
    "AutoConfig": "configuration_ogma.OgmaConfig",
    "AutoModel": "ogma_model.OgmaModel"
  },
  "variant": "transformer",
  "d_embed": 128,
  "d_model": 256,
  "d_output": 256,
  "n_layers": 12,
  "n_heads": 4,
  "vocab_size": 30000,
  "max_seq_len": 1024,
  "matryoshka_dims": [
    32,
    64,
    128,
    256
  ],
  "pooling": "mean",
  "ffn_mult": 2.6666666666666665,
  "conv_kernel_size": 7,
  "spatial_rank": 32,
  "n_random_features": 128,
  "dropout": 0.0,
  "pad_id": 0,
  "unk_id": 1,
  "bos_id": 2,
  "eos_id": 3,
  "qry_id": 4,
  "doc_id": 5,
  "sym_id": 6,
  "n_special_tokens": 7
}