INNER CODE UNIT · Python

LoRAMoeLayer

Leeroo-AI/mergoo · mergoo/compose_layers.py:86

class LoRAMoeLayer(torch.nn.Module):
    def __init__(self, config, in_features, out_features, bias) -> None:
        super().__init__()

        self.config = config
        self.num_experts_per_tok = config.num_experts_per_tok
        self.num_experts = config.num_experts
        self.in_features = in_features
        self.out_features = out_features

        self.r = {}
        self.lora_alpha = {}
        self.scaling = {}
        self.use_dora = {}
        self.lora_dropout = nn.ModuleDict({})
        self.lora_A = nn.ModuleDict({})
        self.lora_B = nn.ModuleDict({})
        self.base_layer = nn.Linear(self.in_features, self.out_features, bias=bias)

View source record →