addressed colin's comments

bibliography.bib

@@ -127,3 +127,67 @@
   year={1912},
   publisher={Wiley Online Library}
 }
+@misc{albalak2024survey,
+      title={A Survey on Data Selection for Language Models},
+      author={Alon Albalak and Yanai Elazar and Sang Michael Xie and Shayne Longpre and Nathan Lambert and Xinyi Wang and Niklas Muennighoff and Bairu Hou and Liangming Pan and Haewon Jeong and Colin Raffel and Shiyu Chang and Tatsunori Hashimoto and William Yang Wang},
+      year={2024},
+      eprint={2402.16827},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{longpre2023pretrainers,
+      title={A Pretrainer's Guide to Training Data: Measuring the Effects of Data Age, Domain Coverage, Quality, & Toxicity},
+      author={Shayne Longpre and Gregory Yauney and Emily Reif and Katherine Lee and Adam Roberts and Barret Zoph and Denny Zhou and Jason Wei and Kevin Robinson and David Mimno and Daphne Ippolito},
+      year={2023},
+      eprint={2305.13169},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{wenzek2019ccnet,
+      title={CCNet: Extracting High Quality Monolingual Datasets from Web Crawl Data},
+      author={Guillaume Wenzek and Marie-Anne Lachaux and Alexis Conneau and Vishrav Chaudhary and Francisco Guzmán and Armand Joulin and Edouard Grave},
+      year={2019},
+      eprint={1911.00359},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{soldaini2024dolma,
+      title={Dolma: an Open Corpus of Three Trillion Tokens for Language Model Pretraining Research},
+      author={Luca Soldaini and Rodney Kinney and Akshita Bhagia and Dustin Schwenk and David Atkinson and Russell Authur and Ben Bogin and Khyathi Chandu and Jennifer Dumas and Yanai Elazar and Valentin Hofmann and Ananya Harsh Jha and Sachin Kumar and Li Lucy and Xinxi Lyu and Nathan Lambert and Ian Magnusson and Jacob Morrison and Niklas Muennighoff and Aakanksha Naik and Crystal Nam and Matthew E. Peters and Abhilasha Ravichander and Kyle Richardson and Zejiang Shen and Emma Strubell and Nishant Subramani and Oyvind Tafjord and Pete Walsh and Luke Zettlemoyer and Noah A. Smith and Hannaneh Hajishirzi and Iz Beltagy and Dirk Groeneveld and Jesse Dodge and Kyle Lo},
+      year={2024},
+      eprint={2402.00159},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{ouyang2022training,
+      title={Training language models to follow instructions with human feedback},
+      author={Long Ouyang and Jeff Wu and Xu Jiang and Diogo Almeida and Carroll L. Wainwright and Pamela Mishkin and Chong Zhang and Sandhini Agarwal and Katarina Slama and Alex Ray and John Schulman and Jacob Hilton and Fraser Kelton and Luke Miller and Maddie Simens and Amanda Askell and Peter Welinder and Paul Christiano and Jan Leike and Ryan Lowe},
+      year={2022},
+      eprint={2203.02155},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{hoffmann2022training,
+      title={Training Compute-Optimal Large Language Models},
+      author={Jordan Hoffmann and Sebastian Borgeaud and Arthur Mensch and Elena Buchatskaya and Trevor Cai and Eliza Rutherford and Diego de Las Casas and Lisa Anne Hendricks and Johannes Welbl and Aidan Clark and Tom Hennigan and Eric Noland and Katie Millican and George van den Driessche and Bogdan Damoc and Aurelia Guy and Simon Osindero and Karen Simonyan and Erich Elsen and Jack W. Rae and Oriol Vinyals and Laurent Sifre},
+      year={2022},
+      eprint={2203.15556},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{muennighoff2023scaling,
+      title={Scaling Data-Constrained Language Models},
+      author={Niklas Muennighoff and Alexander M. Rush and Boaz Barak and Teven Le Scao and Aleksandra Piktus and Nouamane Tazi and Sampo Pyysalo and Thomas Wolf and Colin Raffel},
+      year={2023},
+      eprint={2305.16264},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{hernandez2022scaling,
+      title={Scaling Laws and Interpretability of Learning from Repeated Data},
+      author={Danny Hernandez and Tom Brown and Tom Conerly and Nova DasSarma and Dawn Drain and Sheer El-Showk and Nelson Elhage and Zac Hatfield-Dodds and Tom Henighan and Tristan Hume and Scott Johnston and Ben Mann and Chris Olah and Catherine Olsson and Dario Amodei and Nicholas Joseph and Jared Kaplan and Sam McCandlish},
+      year={2022},
+      eprint={2205.10487},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}

index.html

@@ -168,11 +168,10 @@
     <d-contents>
     </d-contents>
 
-    <!-- Your JavaScript file -->
-
     <p>We have recently released 🍷FineWeb, our new large scale
-        (15T tokens, 44TB disk space) dataset of clean text sourced from the web for LLM pretraining. You can
+        (15T gpt2 tokens, 44TB disk space) dataset of clean text sourced from the web for LLM pretraining. You can
         download it <a href="https://huggingface.co/datasets/HuggingFaceFW/fineweb">here</a>.</p>
+    <p>[TODO: ADD MORE INTRODUCTION]</p>
     <p>As 🍷FineWeb has gathered a lot of interest from the
         community, we decided to further explain the steps involved in creating it, our processing decisions and
         some lessons learned along the way. Read on for all the juicy details on large text dataset creation!</p>
@@ -199,7 +198,7 @@
         They have been crawling the web since 2007 (long before LLMs were a thing) and release a new dump usually
         every 1 or 2 months, which can be freely downloaded. </p>
     <p>As an example, their latest crawl (2024-10) contains 3.16
-        billion web pages, totaling 424.7 TiB of uncompressed content (the size changes from dump to dump). There
+        billion web pages, totaling 424.7 TiB of uncompressed HTML text content (the size changes from dump to dump). There
         are 95 dumps since 2013 and 3 dumps from 2008 to 2012, which are in a different (older) format.<d-footnote>We have not processed these 3 older dumps.</d-footnote> </p>
     <h3>Processing at scale</h3>
     <p>Given the sheer size of the data involved, one of the main
@@ -213,20 +212,19 @@
                 href="https://github.com/huggingface/datatrove">library</a>.</p>
     <h3>What is clean, good data?</h3>
     <p>This is probably the main question to keep in mind when
-        creating a dataset. A good first lesson is that data that would intuitively be considered high quality by a
-        human may not be necessarily the best data (or at least not all that you need) to train a good model on.</p>
+        creating a dataset. In the context of large language model pretraining, "high quality" is not a very well defined term<d-cite bibtex-key="albalak2024survey"></d-cite>, and often not a property of documents that can be easily perceived through direct observation alone.<d-cite bibtex-key="longpre2023pretrainers"></d-cite></p>
     <p>It is still common to train a model on a given corpus
         (wikipedia, or some other web dataset considered clean) and use it to check the perplexity on the dataset
-        that we were trying to curate. Unfortunately this does not always correlate with performance on downstream
-        tasks, and so another often used approach is to train small models (small because training models is
-        expensive and time consuming, and we want to be able to quickly iterate) on our dataset and evaluate them on
+        that we were trying to curate<d-cite bibtex-key="wenzek2019ccnet"></d-cite>. Unfortunately this does not always correlate with performance on downstream
+        tasks<d-cite bibtex-key="soldaini2024dolma"></d-cite>, and so another often used approach is to train small models (small because training models is
+        expensive and time consuming, and we want to be able to quickly iterate) on a representative subset of our dataset and evaluate them on
         a set of evaluation tasks. As we are curating a dataset for pretraining a generalist LLM, it is important to
         choose a diverse set of tasks and try not to overfit to any one individual benchmark.</p>
     <p>Another way to evaluate different datasets would be to
         train a model on each one and have humans rate and compare the outputs of each one (like on the <a
                 href="https://chat.lmsys.org/">LMSYS Chatbot Arena</a>)<d-cite bibtex-key="chiang2024chatbot"></d-cite>. This would arguably provide the most
         reliable results in terms of representing real model usage, but getting ablation results this way is too
-        expensive and slow.</p>
+        expensive and slow. It also often requires that the models have undergone at least an instruction finetuning stage, as pretrained models have difficulty following instructions.<d-cite bibtex-key="ouyang2022training"></d-cite></p>
     <p>The approach we ultimately went with was to train small
         models and evaluate them on a set of benchmark tasks. We believe this is a reasonable proxy for the quality
         of the data used to train these models.</p>
@@ -234,14 +232,14 @@
     <p>To be able to compare the impact of a given processing
         step, we would train 2 models, one where the data included the extra step and another where this step was
         ablated (cut/removed). These 2 models would have the same number of parameters, architecture, and be trained
-        on an equal number of tokens and with the same hyperparameters — the only difference would be in the
+        on an equal number of randomly sampled tokens from each step's data, for a single epoch, and with the same hyperparameters — the only difference would be in the
         training data. We would then evaluate each model on the same set of tasks and compare the average
         scores.</p>
     <p>Our ablation models were trained using <a
             href="https://github.com/huggingface/nanotron"><code>nanotron</code></a> with this config [<strong>TODO:
         INSERT SIMPLIFIED NANOTRON CONFIG HERE</strong>]. The models had 1.82B parameters, used the Llama
         architecture with a 2048 sequence length, and a global batch size of ~2 million tokens. For filtering
-        ablations we mostly trained on ~28B tokens (which is roughly the Chinchilla optimal training size for this
+        ablations we mostly trained on ~28B tokens (which is roughly the Chinchilla<d-cite bibtex-key="hoffmann2022training"></d-cite> optimal training size for this
         model size).</p>
     <p>We evaluated the models using <a
             href="https://github.com/huggingface/lighteval/"><code>lighteval</code></a>. We tried selecting
@@ -281,15 +279,16 @@
         starting point. In our experience the default text extraction (extracting the main text of a webpage from
         its HTML) used to create these WET files is suboptimal and there are a variety of open-source libraries that
         provide better text extraction (by, namely, keeping less boilerplate content/navigation menus). We extracted
-        the text content from the WARC files using the trafilatura library<d-cite bibtex-key="barbaresi-2021-trafilatura"></d-cite>. It is important to note, however, that text extraction is one of the most costly steps of our
-        processing, so we believe that using the readily available WET data could be a reasonable trade-off for
-        lower budget teams.</p>
+        the text content from the WARC files using the trafilatura library<d-cite bibtex-key="barbaresi-2021-trafilatura"></d-cite>, which from visual inspection of the results provided good quality extraction when compared to other libraries.</p><aside>You can also find a benchmark on text extraction libraries <a href="https://github.com/scrapinghub/article-extraction-benchmark/blob/master/README.rst">here</a>.</aside>
     <p>To validate this decision, we processed the 2019-18 dump
-        directly using the WET files and with text extracted from WARC files using trafilatura. We applied the same
+        directly using the WET files and with text extracted from WARC files using trafilatura<d-footnote>We used trafilatura default options with <code>favour_precision=True</code>.</d-footnote>. We applied the same
         processing to each one (our base filtering+minhash, detailed below) and trained two models. While the
-        resulting dataset is considerably larger for the WET data (around 254BT), it proves to be of much worse
-        quality than the one that used trafilatura to extract text from WARC files (which is around 200BT). Many of
+        resulting dataset is about 25% larger for the WET data (around 254 billion tokens), it proves to be of much worse
+        quality than the one that used trafilatura to extract text from WARC files (which is around 200 billion tokens). Visual inspection of some samples confirms that many of
         these additional tokens on the WET files are unnecessary page boilerplate.</p>
+    <p>It is important to note, however, that text extraction is one of the most costly steps of our
+        processing, so we believe that using the readily available WET data could be a reasonable trade-off for
+        lower budget teams.</p>
     <div class="main-plot-container">
         <figure><img src="plots/wet_comparison.png"/></figure>
         <div id="plot-wet_comparison"></div>
@@ -330,7 +329,7 @@
     <p>Removing these duplicates (deduplicating) has been linked to an improvement in model performance<d-cite bibtex-key="lee2022deduplicating"></d-cite> and a reduction in memorization of pretraining data<d-cite bibtex-key="carlini2023quantifying"></d-cite>, which might
         allow for better generalization. Additionally, the performance uplift can also be tied to increased training
         efficiency: by removing duplicated content, for the same number of training tokens, a model will have seen
-        more diverse data.</p>
+        more diverse data.<d-cite bibtex-key="muennighoff2023scaling"></d-cite><d-cite bibtex-key="hernandez2022scaling"></d-cite></p>
     <p>There are different ways to identify and even define
         duplicated data. Common approaches rely on hashing techniques to speed up the process, or on building
         efficient data structures to index the data (like suffix arrays). Methods can also be “fuzzy”, by using some
@@ -338,19 +337,20 @@
         documents (or lines, paragraphs, or whatever other granularity level being used).</p>
     <h4>Our deduplication parameters</h4>
     <p>Similarly to RefinedWeb, we decided to apply MinHash, a
-        fuzzy hash based deduplication technique. We chose to compute minhashes on each document’s 5-grams, using
+        fuzzy hash based deduplication technique that scales well and allows us to tune similarity thresholds (by changing the number and size of buckets) and the granularity of the matches (changing the n-gram size). We chose to compute minhashes on each document’s 5-grams, using
         112 hash functions in total, split into 14 buckets of 8 hashes each — targeting documents that are at least
         75% similar. Documents with the same 8 minhashes in any bucket are considered a duplicate of each other.</p>
     <p>This would mean that for two documents with a similarity (<code>s</code>)
         of 0.7, 0.75, 0.8 and 0.85, the probability that they would be identified as duplicates would be 56%, 77%,
         92% and 98.8% respectively ($$1-(1-s^8)^{14}$$). See the plot below for a match probability
         comparison between our setup with 112 hashes and the one from RefinedWeb, with 9000 hashes, divided into 450
-        buckets of 20 hashes (that requires a substantially larger amount of compute resources):</p>
+        buckets of 20 hashes (that requires a substantially larger amount of compute resources, as each individual hash must be computed, stored and then compared with hashes from other documents):</p>
     <figure><img src="plots/minhash_parameters_comparison.png"/>
     </figure>
     <p>While the high number of hash functions in RefinedWeb
-        allows for a steeper, more well defined cut off, we believe the compute and storage savings are a reasonable
+        allows for a steeper, more well defined cut off (documents with real similarity near the threshold are more likely to be correctly identified), we believe the compute and storage savings are a reasonable
         trade off.</p>
+    <p>It should also be noted that intra-document deduplication is already handled by our repetition filter, which removes documents with many repeated lines and paragraphs.</p>
     <h4>More deduplication is always better, right?</h4>
     <p>Our initial approach was to take the entire dataset (all
         95 dumps) and deduplicate them as one big dataset using MinHash.</p>
@@ -381,7 +381,7 @@
             removed)
         </li>
     </ul>
-    <p>As an experiment, we tried training two models on 28BT
+    <p>As an experiment, we tried training two models on 28 billion tokens
         sampled from the following data from 2013-48:</p>
     <ul>
         <li>the fully deduplicated remaining ~31 billion tokens (<em>originally kept
@@ -391,8 +391,10 @@
     <ul>
         <li>171 billion tokens obtained by individually deduplicating (without
             considering the other dumps) the ~460 billion tokens that had been removed from this dump in the
-            iterative dedup process (<em>originally removed data</em>)
+            iterative dedup process (<em>originally removed data</em>)<d-footnote>While there may be documents in <em>originally kept
+            data</em> similar to documents in <em>originally removed data</em>, we estimate the overlap to be small (around 4 billion tokens)</d-footnote>
         </li>
+
     </ul>
     <div class="main-plot-container">
         <figure><img src="plots/removed_data_cross_dedup.png"/></figure>
@@ -400,7 +402,8 @@
     </div>
     <p>These results show that, for this older dump where we were
         removing over 90% of the original data, the data that was kept was actually <em>worse</em> than the data
-        removed (considered independently of all the other dumps).</p>
+        removed (considered independently of all the other dumps). This is also confirmed by visual inspection: <em>originally kept
+            data</em> contains far more ads, lists of keywords and generally badly formatted text than <em>originally removed data</em>.</p>
     <h4>Taking a step back: individual dump dedup</h4>
     <p>We then tried an alternative approach: we deduplicated
         each dump with MinHash individually (without considering the other dumps). This resulted in 20 trillion
@@ -469,9 +472,9 @@
         documents duplicated up to 8 times. This simulation illustrates the inherent difficulties associated with
         measuring deduplication impact on the training of LLMs, once the biggest document clusters have been
         removed.</p>
-    <h4>Other (failed) approaches</h4>
+    <h4>Other (failed) global approaches</h4>
     <p>We attempted to improve the performance of the
-        independently minhash deduped 20T of data by further deduplicating it with the following methods</p>
+        independently minhash deduped 20 trillion tokens of data by further deduplicating it (globally, over all crawls) with the following methods</p>
     <ul>
         <li>URL deduplication, where we only kept one document per normalized
             (lowercased) URL (71.5% of tokens removed, 5.6T left) — <em>FineWeb URL dedup</em></li>
@@ -479,7 +482,7 @@
     <ul>
         <li>Line deduplication:
             <ul>
-                <li>remove all but 1 occurrence of each duplicated line (77.8% of
+                <li>remove all but 1 (randomly chosen) occurrence of each duplicated line (77.8% of
                     tokens dropped, 4.4T left) — <em>FineWeb line dedup</em></li>
             </ul>
             <ul>
@@ -489,7 +492,7 @@
             </ul>
             <ul>
                 <li>remove all but 1 occurrence of each span of 3 duplicated lines
-                    with all numbers replaced by 0 (80.9% of tokens removed, 3.7T left) — <em>FineWeb 3-line
+                    with each number treated as 0 when finding duplicates, (80.9% of tokens removed, 3.7T left) — <em>FineWeb 3-line
                         dedup</em></li>
             </ul>
         </li>
@@ -518,8 +521,7 @@
         benchmark, one of the benchmarks in our “early signal” group with the stronger signal and highest
         signal-over-noise ratio. As such, it has stayed a common sub-set of typical LLM training, for instance in
         the relatively recent Llama1 model<d-cite bibtex-key="touvron2023llama"></d-cite>. We experimented applying
-        each of the different filters used in C4 to a baseline of the independently deduped FineWeb 2019-18 dump
-        (plot smoothed with a 3 checkpoints sliding window):</p>
+        each of the different filters used in C4 to a baseline of the independently deduped FineWeb 2019-18 dump:</p>
     <div class="main-plot-container">
         <figure><img src="plots/c4_filters_hellaswag.png"/></figure>
         <div id="plot-c4_filters_hellaswag"></div>