File size: 4,633 Bytes
3b609b9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 |
# Copyright 2023-present the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import warnings
from dataclasses import dataclass
from typing import Optional
from peft.config import PeftConfig
from peft.utils.peft_types import PeftType
@dataclass
class XLoraConfig(PeftConfig):
r"""
This is the configuration class to store the configuration of a `XLoraModel`. When the config is reloaded, the
paths of the `adapters` field is disregarded in favor of the saved adapters. As such, only the keys matter during
loading.
Args:
hidden_size (`int`):
Hidden size of the base model.
adapters (`dict`):
Mapping of adapter names to the LoRA adapter id, as per PeftModel.load_adapter. *They will be automatically
loaded*, to use as LoRA experts. When using from_pretrained, pass the new adapters dict as a keyword
argument.
enable_softmax (`bool`, *optional*, defaults to `True`):
Enable softmax application for the X-LoRA classifier.
enable_softmax_topk (`bool`, *optional*, defaults to `False`):
Enable softmax application for the top-k LoRA adapters. Mutually exclusive to `enable_softmax` and must
only be set if `top_k_lora` is.
softmax_temperature (`float`, *optional*, defaults to 1.0):
Softmax temperature, lower yields sharper predictions
layerwise_scalings (`bool`, *optional*, defaults to `False`):
If True, generate scalings for each LoRA adapter (each layer). If this is False, then scalings will be
broadcasted, the same, to each layer.
top_k_lora (`int`, *optional*, defaults to None):
Sparsely select the top_k LoRA experts instead of the default dense method.
xlora_depth (`int`, *optional*, defaults to 1):
Depth of the X-LoRA classifier.
xlora_size (`int`, *optional*, defaults to 2048):
Hidden size of the X-LoRA classifier, irrelevant if `xlora_depth=1`.
xlora_dropout_p (`float`, *optional*, defaults to 0.2):
Dropout probability of the X-LoRA classifier, irrelevant if `xlora_depth=1`.
use_trainable_adapters (`bool`, *optional*, defaults to False):
Make the adapters trainable.
scaling_pass_value (`float`, *optional*, defaults to 0):
Scaling pass value.
global_scaling_weight (`float`, *optional*, defaults to 1):
Weight to multiply output of each LoRA adapter by.
"""
hidden_size: int = None # type: ignore
adapters: dict[str, str] = None # type: ignore
enable_softmax: bool = True
enable_softmax_topk: bool = False
layerwise_scalings: bool = False
xlora_depth: int = 1
xlora_size: int = 2048
xlora_dropout_p: float = 0.2
use_trainable_adapters: bool = False
softmax_temperature: float = 1.0
top_k_lora: Optional[int] = None
scaling_pass_value: float = 0.0
global_scaling_weight: float = 1.0
def __post_init__(self):
super().__post_init__()
self.peft_type = PeftType.XLORA
if self.hidden_size is None:
warnings.warn(
"No value was provided for `hidden_size`. This will be set to 4096 by default, please ensure that this is correct."
)
self.hidden_size = 4096
if self.adapters is None:
warnings.warn(
"No value was provided for for `adapters`. This will be set to empty, please ensure that this is correct."
)
self.adapters = {}
if self.enable_softmax_topk and self.top_k_lora is None:
warnings.warn("`enable_softmax_topk` enabled `top_k_lora` is not set")
if self.enable_softmax_topk and self.enable_softmax:
warnings.warn(
"`enable_softmax_topk` and `enable_softmax` are both enabled. This will result in worse performance."
)
if self.top_k_lora is not None and self.top_k_lora < 1:
warnings.warn("`top_k_lora` value must be at least 1.")
|