# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import os
from fairseq import options, utils
from fairseq.data import (
ConcatDataset,
data_utils,
LanguagePairDataset)
from ..data import SubsampleLanguagePairDataset
import logging
from fairseq.tasks import register_task
from fairseq.tasks.translation import TranslationTask, load_langpair_dataset
logger = logging.getLogger(__name__)
def concat_language_pair_dataset(*language_pair_datasets, up_sample_ratio=None,
all_dataset_upsample_ratio=None):
logger.info("To cancat the language pairs")
dataset_number = len(language_pair_datasets)
if dataset_number == 1:
return language_pair_datasets[0]
elif dataset_number < 1:
raise ValueError("concat_language_pair_dataset needs at least on dataset")
# for dataset in language_pair_datasets:
# assert isinstance(dataset, LanguagePairDataset), "concat_language_pair_dataset can only concat language pair" \
# "dataset"
src_list = [language_pair_datasets[0].src]
tgt_list = [language_pair_datasets[0].tgt]
src_dict = language_pair_datasets[0].src_dict
tgt_dict = language_pair_datasets[0].tgt_dict
left_pad_source = language_pair_datasets[0].left_pad_source
left_pad_target = language_pair_datasets[0].left_pad_target
logger.info("To construct the source dataset list and the target dataset list")
for dataset in language_pair_datasets[1:]:
assert dataset.src_dict == src_dict
assert dataset.tgt_dict == tgt_dict
assert dataset.left_pad_source == left_pad_source
assert dataset.left_pad_target == left_pad_target
src_list.append(dataset.src)
tgt_list.append(dataset.tgt)
logger.info("Have constructed the source dataset list and the target dataset list")
if all_dataset_upsample_ratio is None:
sample_ratio = [1] * len(src_list)
sample_ratio[0] = up_sample_ratio
else:
sample_ratio = [int(t) for t in all_dataset_upsample_ratio.strip().split(",")]
assert len(sample_ratio) == len(src_list)
src_dataset = ConcatDataset(src_list, sample_ratios=sample_ratio)
tgt_dataset = ConcatDataset(tgt_list, sample_ratios=sample_ratio)
res = LanguagePairDataset(
src_dataset, src_dataset.sizes, src_dict,
tgt_dataset, tgt_dataset.sizes, tgt_dict,
left_pad_source=left_pad_source,
left_pad_target=left_pad_target,
)
logger.info("Have created the concat language pair dataset")
return res
@register_task('translation_w_mono')
class TranslationWithMonoTask(TranslationTask):
"""
Translate from one (source) language to another (target) language.
Args:
src_dict (~fairseq.data.Dictionary): dictionary for the source language
tgt_dict (~fairseq.data.Dictionary): dictionary for the target language
.. note::
The translation task is compatible with :mod:`fairseq-train`,
:mod:`fairseq-generate` and :mod:`fairseq-interactive`.
The translation task provides the following additional command-line
arguments:
.. argparse::
:ref: fairseq.tasks.translation_parser
:prog:
"""
@staticmethod
def add_args(parser):
"""Add task-specific arguments to the parser."""
# fmt: off
TranslationTask.add_args(parser)
parser.add_argument('--mono-data', default=None, help='monolingual data, split by :')
parser.add_argument('--mono-one-split-each-epoch', action='store_true', default=False, help='use on split of monolingual data at each epoch')
parser.add_argument('--parallel-ratio', default=1.0, type=float, help='subsample ratio of parallel data')
parser.add_argument('--mono-ratio', default=1.0, type=float, help='subsample ratio of mono data')
def __init__(self, args, src_dict, tgt_dict):
super().__init__(args, src_dict, tgt_dict)
self.src_dict = src_dict
self.tgt_dict = tgt_dict
self.update_number = 0
@classmethod
def setup_task(cls, args, **kwargs):
"""Setup the task (e.g., load dictionaries).
Args:
args (argparse.Namespace): parsed command-line arguments
"""
args.left_pad_source = options.eval_bool(args.left_pad_source)
args.left_pad_target = options.eval_bool(args.left_pad_target)
if getattr(args, 'raw_text', False):
utils.deprecation_warning('--raw-text is deprecated, please use --dataset-impl=raw')
args.dataset_impl = 'raw'
elif getattr(args, 'lazy_load', False):
utils.deprecation_warning('--lazy-load is deprecated, please use --dataset-impl=lazy')
args.dataset_impl = 'lazy'
paths = utils.split_paths(args.data)
assert len(paths) > 0
# find language pair automatically
if args.source_lang is None or args.target_lang is None:
args.source_lang, args.target_lang = data_utils.infer_language_pair(paths[0])
if args.source_lang is None or args.target_lang is None:
raise Exception('Could not infer language pair, please provide it explicitly')
# load dictionaries
src_dict = cls.load_dictionary(os.path.join(paths[0], 'dict.{}.txt'.format(args.source_lang)))
tgt_dict = cls.load_dictionary(os.path.join(paths[0], 'dict.{}.txt'.format(args.target_lang)))
assert src_dict.pad() == tgt_dict.pad()
assert src_dict.eos() == tgt_dict.eos()
assert src_dict.unk() == tgt_dict.unk()
logger.info('| [{}] dictionary: {} types'.format(args.source_lang, len(src_dict)))
logger.info('| [{}] dictionary: {} types'.format(args.target_lang, len(tgt_dict)))
return cls(args, src_dict, tgt_dict)
def load_dataset(self, split, epoch=0, combine=False, **kwargs):
"""Load a given dataset split.
Args:
split (str): name of the split (e.g., train, valid, test)
"""
logger.info("To load the dataset {}".format(split))
paths = utils.split_paths(self.args.data)
assert len(paths) > 0
if split != getattr(self.args, "train_subset", None):
# if not training data set, use the first shard for valid and test
paths = paths[:1]
data_path = paths[(epoch - 1) % len(paths)]
mono_paths = utils.split_paths(self.args.mono_data)
# infer langcode
src, tgt = self.args.source_lang, self.args.target_lang
parallel_data = load_langpair_dataset(
data_path, split, src, self.src_dict, tgt, self.tgt_dict,
combine=combine, dataset_impl=self.args.dataset_impl,
upsample_primary=self.args.upsample_primary,
left_pad_source=self.args.left_pad_source,
left_pad_target=self.args.left_pad_target,
max_source_positions=self.args.max_source_positions,
max_target_positions=self.args.max_target_positions,
load_alignments=self.args.load_alignments,
num_buckets=self.args.num_batch_buckets,
shuffle=(split != "test"),
pad_to_multiple=self.args.required_seq_len_multiple,
)
if split == "train":
parallel_data = SubsampleLanguagePairDataset(parallel_data, size_ratio=self.args.parallel_ratio,
seed=self.args.seed,
epoch=epoch)
if self.args.mono_one_split_each_epoch:
mono_path = mono_paths[(epoch - 1) % len(mono_paths)] # each at one epoch
mono_data = load_langpair_dataset(
mono_path, split, src, self.src_dict, tgt, self.tgt_dict,
combine=combine, dataset_impl=self.args.dataset_impl,
upsample_primary=self.args.upsample_primary,
left_pad_source=self.args.left_pad_source,
left_pad_target=self.args.left_pad_target,
max_source_positions=self.args.max_source_positions,
shuffle=(split != "test"),
max_target_positions=self.args.max_target_positions,
)
mono_data = SubsampleLanguagePairDataset(mono_data, size_ratio=self.args.mono_ratio,
seed=self.args.seed,
epoch=epoch)
all_dataset = [parallel_data, mono_data]
else:
mono_datas = []
for mono_path in mono_paths:
mono_data = load_langpair_dataset(
mono_path, split, src, self.src_dict, tgt, self.tgt_dict,
combine=combine, dataset_impl=self.args.dataset_impl,
upsample_primary=self.args.upsample_primary,
left_pad_source=self.args.left_pad_source,
left_pad_target=self.args.left_pad_target,
max_source_positions=self.args.max_source_positions,
shuffle=(split != "test"),
max_target_positions=self.args.max_target_positions,
)
mono_data = SubsampleLanguagePairDataset(mono_data, size_ratio=self.args.mono_ratio,
seed=self.args.seed,
epoch=epoch)
mono_datas.append(mono_data)
all_dataset = [parallel_data] + mono_datas
self.datasets[split] = ConcatDataset(all_dataset)
else:
self.datasets[split] = parallel_data