sample from normal dist for R and X denoisers

app.py

@@ -32,7 +32,7 @@ def mask_spans(
         # n = 1
         mu = max(1, int(len(tokens) * r))
         start = max(
-            0, len(tokens) - random.randint(1, int(2 * mu))
+            0, len(tokens) - random.randint(1, int(2 * mu))  # sample from uniform distribution for S denoisers
         )  # max to handle start < 0
         encoder_inputs += tokens[:start] + [sentinel_id]
         encoder_mask += ([1] * len(tokens[:start])) + [0]
@@ -47,8 +47,10 @@ def mask_spans(
         start = 0
         end = 0
         while start < len(tokens):
-            # uniform random span length
-            length = random.randint(1, int(2 * mu))
+            # for R and X denoisers, sample random span length from normal distribution bounded from 1 to 2 * mu.
+            # std of 0.25 * mu is arbitrary, not specified in paper but makes a sane looking distribution
+            #  at extreme ends of span length means (from 3 to 64).
+            length = max(1, min(int(2 * mu), int(np.round(np.random.normal(mu, 0.25 * mu)))))
             end = min(start + length, len(tokens))
 
             # randomly decide if span should be masked