config.json

{
  "architectures": [
    "GPTNeoXCodebookModel"
  ],
  "codebook_at": [
    "attn_preproj"
  ],
  "codebook_type": "group",
  "k_codebook": 8,
  "kmeans_init": false,
  "kmeans_init_examples": 1000,
  "kmeans_kwargs": {
    "batch_size": 24576,
    "n_init": "auto"
  },
  "kmeans_path": "/.cache/cb_volume/huggingface/kmeans_embeddings.pt",
  "layers_to_snap": [
    0,
    1,
    2,
    3,
    4,
    5,
    6,
    7,
    8,
    9,
    10,
    11,
    12,
    13,
    14,
    15,
    16,
    17,
    18,
    19,
    20,
    21,
    22,
    23
  ],
  "loss": "aeloss",
  "model_type": "codebook",
  "num_codebooks": 16,
  "num_codes": 10000,
  "similarity_metric": "inner_product",
  "torch_dtype": "float32",
  "transformers_version": "4.27.3"
}