INNER CODE UNIT · Python
result
Leeroo-AI/mergoo · mergoo/compose_layers.py:133
result = self.base_layer(x, *args, **kwargs)
"""TODO MAYBE
- tensorize this loop add learnable weights here
- These are in my mind ( sigle embedding, each lora layer with a gate, lora gating loss similar to iclr )
"""
for ix, active_adapter in enumerate(self.active_adapters):
if active_adapter not in self.lora_A.keys():
continue
lora_A = self.lora_A[active_adapter]
lora_B = self.lora_B[active_adapter]
dropout = self.lora_dropout[active_adapter]
scaling = self.scaling[active_adapter]
x = x.to(lora_A.weight.dtype) # type: ignore
batch_idx, tok_idx, expert_idx = torch.where(selected_experts == ix)
x_adapter = x[