Update model.py
Browse files
model.py
CHANGED
@@ -349,17 +349,17 @@ class rotary(nn.Module):
|
|
349 |
|
350 |
if self.radii:
|
351 |
radius = self.align_f0(ctx)
|
352 |
-
if "rotary2" in self.debug:
|
353 |
print(f"{layer} radius: {radius} ctx: {ctx}")
|
354 |
else:
|
355 |
radius = freqs
|
356 |
freqs = torch.polar(torch.ones_like(radius), freqs)
|
357 |
|
358 |
-
if "rotary3" in self.debug:
|
359 |
print(f"{layer} count {self._counter} f0: {f0.shape if f0 is not None else None} freqs: {freqs.shape} radius: {radius.shape} ctx: {ctx}")
|
360 |
print(f"freqs mean: {freqs.mean():.2f} inv_freq mean: {self.inv_freq.mean():.2f} theta: {self.theta.item():.2f} radius mean: {radius.mean():.2f} radius shape: {radius.shape} ctx: {ctx}")
|
361 |
|
362 |
-
if "rotary_detail" in self.debug:
|
363 |
print(f"\n==== Detailed RoPE Analysis ====")
|
364 |
print(f"Layer: {layer}, Context Length: {ctx}")
|
365 |
print(f"F0 stats: mean={self.theta.item():.2f}")
|
|
|
349 |
|
350 |
if self.radii:
|
351 |
radius = self.align_f0(ctx)
|
352 |
+
if "rotary2" in self.debug and self._counter == 5:
|
353 |
print(f"{layer} radius: {radius} ctx: {ctx}")
|
354 |
else:
|
355 |
radius = freqs
|
356 |
freqs = torch.polar(torch.ones_like(radius), freqs)
|
357 |
|
358 |
+
if "rotary3" in self.debug and self._counter == 5:
|
359 |
print(f"{layer} count {self._counter} f0: {f0.shape if f0 is not None else None} freqs: {freqs.shape} radius: {radius.shape} ctx: {ctx}")
|
360 |
print(f"freqs mean: {freqs.mean():.2f} inv_freq mean: {self.inv_freq.mean():.2f} theta: {self.theta.item():.2f} radius mean: {radius.mean():.2f} radius shape: {radius.shape} ctx: {ctx}")
|
361 |
|
362 |
+
if "rotary_detail" in self.debug and self._counter == 5:
|
363 |
print(f"\n==== Detailed RoPE Analysis ====")
|
364 |
print(f"Layer: {layer}, Context Length: {ctx}")
|
365 |
print(f"F0 stats: mean={self.theta.item():.2f}")
|