# Copyright 2023 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Funnel Transformer network.""" # pylint: disable=g-classes-have-attributes import math from typing import Any, Callable, Optional, Sequence, Union from absl import logging import numpy as np import tensorflow as tf, tf_keras from official.modeling import tf_utils from official.nlp.modeling import layers _Initializer = Union[str, tf_keras.initializers.Initializer] _Activation = Union[str, Callable[..., Any]] _MAX = 'max' _AVG = 'avg' _TRUNCATED_AVG = 'truncated_avg' _transformer_cls2str = { layers.TransformerEncoderBlock: 'TransformerEncoderBlock', layers.ReZeroTransformer: 'ReZeroTransformer' } _str2transformer_cls = { 'TransformerEncoderBlock': layers.TransformerEncoderBlock, 'ReZeroTransformer': layers.ReZeroTransformer } _approx_gelu = lambda x: tf_keras.activations.gelu(x, approximate=True) def _get_policy_dtype(): try: return tf_keras.mixed_precision.global_policy().compute_dtype or tf.float32 except AttributeError: # tf1 has no attribute 'global_policy' return tf.float32 def _pool_and_concat(mask, unpool_length: int, strides: Union[Sequence[int], int], axes: Union[Sequence[int], int]): """Pools the mask along a given axis with stride. It also skips first unpool_length elements. Args: mask: Tensor to be pooled. unpool_length: Leading elements to be skipped. strides: Strides for the given axes. axes: Axes to pool the Tensor. Returns: Pooled and concatenated Tensor. """ # Wraps the axes as a list. if isinstance(axes, int): axes = [axes] if isinstance(strides, int): strides = [strides] * len(axes) else: if len(strides) != len(axes): raise ValueError('The lengths of strides and axes need to match.') # Bypass no pooling cases. if np.all(np.array(strides) == 1): return mask for axis, stride in zip(axes, strides): # Skips first `unpool_length` tokens. unpool_tensor_shape = [slice(None)] * axis + [slice(None, unpool_length)] unpool_tensor = mask[unpool_tensor_shape] # Pools the second half. pool_tensor_shape = [slice(None)] * axis + [ slice(unpool_length, None, stride) ] pool_tensor = mask[pool_tensor_shape] mask = tf.concat((unpool_tensor, pool_tensor), axis=axis) return mask def _create_fractional_pool_transform(sl: int, pool_factor: float): """Create pooling transform for fractional pooling factor.""" assert pool_factor > 1.0, '`pool_factor` should be > 1.0.' psl = int(sl / pool_factor) gcd_ = math.gcd(sl, psl) # It is expected chunk_sl and chunk_psl are small integers. # The transform is built by tiling a [chunk_sl, chunk_psl] submatrix # gcd_ times. The submatrix sums to chunk_psl. chunk_sl = sl // gcd_ chunk_psl = psl // gcd_ num_one_entries = chunk_psl - 1 num_frac_entries = chunk_sl - (chunk_psl - 1) # The transform is of shape [sl, psl]. transform = np.zeros((sl, psl)) for i in range(sl // chunk_sl): row_start = chunk_sl * i col_start = chunk_psl * i for idx in range(num_one_entries): transform[row_start + idx][col_start + idx] = 1.0 for idx in range(num_frac_entries): transform[row_start + num_one_entries + idx][ col_start + num_one_entries ] = (1.0 / num_frac_entries) return tf.constant(transform, dtype=_get_policy_dtype()) def _create_truncated_avg_transforms( seq_length: int, pool_strides: Sequence[int] ): """Computes pooling transforms. The pooling_transform is of shape [seq_length, seq_length//pool_stride] and pooling_transform[i,j] = 1.0/pool_stride if i//pool_stride == j 0.0 otherwise. It's in essense average pooling but truncate the final window if it seq_length % pool_stride != 0. For seq_length==6 and pool_stride==2, it is [[ 0.5, 0.0, 0.0 ], [ 0.5, 0.0, 0.0 ], [ 0.0, 0.5, 0.0 ], [ 0.0, 0.5, 0.0 ], [ 0.0, 0.0, 0.5 ], [ 0.0, 0.0, 0.5 ]] Args: seq_length: int, sequence length. pool_strides: Sequence of pooling strides for each layer. Returns: pooling_transforms: Sequence of pooling transforms (Tensors) for each layer. """ pooling_transforms = [] for pool_stride in pool_strides: if pool_stride == 1: pooling_transforms.append(None) else: pooled_seq_length = int(seq_length / pool_stride) if (1.0 * pool_stride).is_integer(): pfac, sl, psl = pool_stride, seq_length, pooled_seq_length transform = [ [1.0 if (i // pfac) == j else 0.0 for j in range(psl)] for i in range(sl) ] transform = ( tf.constant(transform, dtype=_get_policy_dtype()) / pool_stride ) else: transform = _create_fractional_pool_transform(seq_length, pool_stride) pooling_transforms.append(transform) seq_length = pooled_seq_length return pooling_transforms def _create_truncated_avg_masks(input_mask: tf.Tensor, pool_strides: Sequence[int], transforms: Sequence[tf.Tensor]): """Computes attention masks. For [1,1,1,0,0] Args: input_mask: Tensor of shape [batch_size, seq_length]. pool_strides: Sequence of pooling strides for each layer. transforms: Sequence of off-diagonal matrices filling with 0.0 and 1/pool_stride. Returns: attention_masks: Sequence of attention masks for each layer. """ def create_2d_mask(from_length, mask): return tf.einsum('F,BT->BFT', tf.ones([from_length], dtype=mask.dtype), mask) attention_masks = [] seq_length = tf.shape(input_mask)[-1] layer_mask = tf.cast(input_mask, dtype=_get_policy_dtype()) for pool_stride, transform in zip(pool_strides, transforms): if pool_stride == 1: attention_masks.append(create_2d_mask(seq_length, layer_mask)) else: pooled_seq_length = tf.cast( tf.cast(seq_length, tf.float32) / tf.cast(pool_stride, tf.float32), tf.int32, ) attention_masks.append(create_2d_mask(pooled_seq_length, layer_mask)) layer_mask = tf.cast( tf.einsum('BF,FT->BT', layer_mask, transform) > 0.0, dtype=layer_mask.dtype, ) seq_length = pooled_seq_length del seq_length return attention_masks @tf_keras.utils.register_keras_serializable(package='Text') class FunnelTransformerEncoder(tf_keras.layers.Layer): """Funnel Transformer-based encoder network. Funnel Transformer Implementation of https://arxiv.org/abs/2006.03236. This implementation utilizes the base framework with Bert (https://arxiv.org/abs/1810.04805). Its output is compatible with `BertEncoder`. Args: vocab_size: The size of the token vocabulary. hidden_size: The size of the transformer hidden layers. num_layers: The number of transformer layers. num_attention_heads: The number of attention heads for each transformer. The hidden size must be divisible by the number of attention heads. max_sequence_length: The maximum sequence length that this encoder can consume. If None, max_sequence_length uses the value from sequence length. This determines the variable shape for positional embeddings. type_vocab_size: The number of types that the 'type_ids' input can take. inner_dim: The output dimension of the first Dense layer in a two-layer feedforward network for each transformer. inner_activation: The activation for the first Dense layer in a two-layer feedforward network for each transformer. output_dropout: Dropout probability for the post-attention and output dropout. attention_dropout: The dropout rate to use for the attention layers within the transformer layers. pool_type: Pooling type. Choose from ['max', 'avg', 'truncated_avg']. pool_stride: An int or a list of ints. Pooling stride(s) to compress the sequence length. If set to int, each layer will have the same stride size. If set to list, the number of elements needs to match num_layers. unpool_length: Leading n tokens to be skipped from pooling. initializer: The initialzer to use for all weights in this encoder. output_range: The sequence output range, [0, output_range), by slicing the target sequence of the last transformer layer. `None` means the entire target sequence will attend to the source sequence, which yields the full output. embedding_width: The width of the word embeddings. If the embedding width is not equal to hidden size, embedding parameters will be factorized into two matrices in the shape of ['vocab_size', 'embedding_width'] and ['embedding_width', 'hidden_size'] ('embedding_width' is usually much smaller than 'hidden_size'). embedding_layer: An optional Layer instance which will be called to generate embeddings for the input word IDs. norm_first: Whether to normalize inputs to attention and intermediate dense layers. If set False, output of attention and intermediate dense layers is normalized. This does not apply to ReZero. transformer_cls: str or a keras Layer. This is the base TransformerBlock the funnel encoder relies on. share_rezero: bool. Whether to share ReZero alpha between the attention layer and the ffn layer. This option is specific to ReZero. with_dense_inputs: Whether to accept dense embeddings as the input. """ def __init__( self, vocab_size: int, hidden_size: int = 768, num_layers: int = 12, num_attention_heads: int = 12, max_sequence_length: int = 512, type_vocab_size: int = 16, inner_dim: int = 3072, inner_activation: _Activation = _approx_gelu, output_dropout: float = 0.1, attention_dropout: float = 0.1, pool_type: str = _MAX, pool_stride: Union[int, Sequence[Union[int, float]]] = 2, unpool_length: int = 0, initializer: _Initializer = tf_keras.initializers.TruncatedNormal( stddev=0.02 ), output_range: Optional[int] = None, embedding_width: Optional[int] = None, embedding_layer: Optional[tf_keras.layers.Layer] = None, norm_first: bool = False, transformer_cls: Union[ str, tf_keras.layers.Layer ] = layers.TransformerEncoderBlock, share_rezero: bool = False, append_dense_inputs: bool = False, **kwargs ): super().__init__(**kwargs) if output_range is not None: logging.warning('`output_range` is available as an argument for `call()`.' 'The `output_range` as __init__ argument is deprecated.') activation = tf_keras.activations.get(inner_activation) initializer = tf_keras.initializers.get(initializer) if embedding_width is None: embedding_width = hidden_size if embedding_layer is None: self._embedding_layer = layers.OnDeviceEmbedding( vocab_size=vocab_size, embedding_width=embedding_width, initializer=tf_utils.clone_initializer(initializer), name='word_embeddings') else: self._embedding_layer = embedding_layer self._position_embedding_layer = layers.PositionEmbedding( initializer=tf_utils.clone_initializer(initializer), max_length=max_sequence_length, name='position_embedding') self._type_embedding_layer = layers.OnDeviceEmbedding( vocab_size=type_vocab_size, embedding_width=embedding_width, initializer=tf_utils.clone_initializer(initializer), use_one_hot=True, name='type_embeddings') self._embedding_norm_layer = tf_keras.layers.LayerNormalization( name='embeddings/layer_norm', axis=-1, epsilon=1e-12, dtype=tf.float32) self._embedding_dropout = tf_keras.layers.Dropout( rate=output_dropout, name='embedding_dropout') # We project the 'embedding' output to 'hidden_size' if it is not already # 'hidden_size'. self._embedding_projection = None if embedding_width != hidden_size: self._embedding_projection = tf_keras.layers.EinsumDense( '...x,xy->...y', output_shape=hidden_size, bias_axes='y', kernel_initializer=tf_utils.clone_initializer(initializer), name='embedding_projection') self._transformer_layers = [] self._attention_mask_layer = layers.SelfAttentionMask( name='self_attention_mask') # Will raise an error if the string is not supported. if isinstance(transformer_cls, str): transformer_cls = _str2transformer_cls[transformer_cls] self._num_layers = num_layers for i in range(num_layers): layer = transformer_cls( num_attention_heads=num_attention_heads, intermediate_size=inner_dim, inner_dim=inner_dim, intermediate_activation=inner_activation, inner_activation=inner_activation, output_dropout=output_dropout, attention_dropout=attention_dropout, norm_first=norm_first, kernel_initializer=tf_utils.clone_initializer(initializer), share_rezero=share_rezero, name='transformer/layer_%d' % i) self._transformer_layers.append(layer) self._pooler_layer = tf_keras.layers.Dense( units=hidden_size, activation='tanh', kernel_initializer=tf_utils.clone_initializer(initializer), name='pooler_transform') if isinstance(pool_stride, int): # TODO(b/197133196): Pooling layer can be shared. pool_strides = [pool_stride] * num_layers else: if len(pool_stride) != num_layers: raise ValueError('Lengths of pool_stride and num_layers are not equal.') pool_strides = pool_stride is_fractional_pooling = False in [ (1.0 * pool_stride).is_integer() for pool_stride in pool_strides ] if is_fractional_pooling and pool_type in [_MAX, _AVG]: raise ValueError( 'Fractional pooling is only supported for' ' `pool_type`=`truncated_average`' ) # TODO(crickwu): explore tf_keras.layers.serialize method. if pool_type == _MAX: pool_cls = tf_keras.layers.MaxPooling1D elif pool_type == _AVG: pool_cls = tf_keras.layers.AveragePooling1D elif pool_type == _TRUNCATED_AVG: # TODO(b/203665205): unpool_length should be implemented. if unpool_length != 0: raise ValueError('unpool_length is not supported by truncated_avg now.') else: raise ValueError('pool_type not supported.') if pool_type in (_MAX, _AVG): self._att_input_pool_layers = [] for layer_pool_stride in pool_strides: att_input_pool_layer = pool_cls( pool_size=layer_pool_stride, strides=layer_pool_stride, padding='same', name='att_input_pool_layer') self._att_input_pool_layers.append(att_input_pool_layer) self._max_sequence_length = max_sequence_length self._pool_strides = pool_strides # This is a list here. self._unpool_length = unpool_length self._pool_type = pool_type self._append_dense_inputs = append_dense_inputs self._config = { 'vocab_size': vocab_size, 'hidden_size': hidden_size, 'num_layers': num_layers, 'num_attention_heads': num_attention_heads, 'max_sequence_length': max_sequence_length, 'type_vocab_size': type_vocab_size, 'inner_dim': inner_dim, 'inner_activation': tf_keras.activations.serialize(activation), 'output_dropout': output_dropout, 'attention_dropout': attention_dropout, 'initializer': tf_keras.initializers.serialize(initializer), 'output_range': output_range, 'embedding_width': embedding_width, 'embedding_layer': embedding_layer, 'norm_first': norm_first, 'pool_type': pool_type, 'pool_stride': pool_stride, 'unpool_length': unpool_length, 'transformer_cls': _transformer_cls2str.get( transformer_cls, str(transformer_cls) ), } self.inputs = dict( input_word_ids=tf_keras.Input(shape=(None,), dtype=tf.int32), input_mask=tf_keras.Input(shape=(None,), dtype=tf.int32), input_type_ids=tf_keras.Input(shape=(None,), dtype=tf.int32)) def call(self, inputs, output_range: Optional[tf.Tensor] = None): # inputs are [word_ids, mask, type_ids] word_embeddings = None if isinstance(inputs, (list, tuple)): logging.warning('List inputs to %s are discouraged.', self.__class__) if len(inputs) == 3: word_ids, mask, type_ids = inputs dense_inputs = None dense_mask = None dense_type_ids = None elif len(inputs) == 6: word_ids, mask, type_ids, dense_inputs, dense_mask, dense_type_ids = ( inputs ) else: raise ValueError( 'Unexpected inputs to %s with length at %d.' % (self.__class__, len(inputs)) ) elif isinstance(inputs, dict): word_ids = inputs.get('input_word_ids') mask = inputs.get('input_mask') type_ids = inputs.get('input_type_ids') word_embeddings = inputs.get('input_word_embeddings', None) dense_inputs = inputs.get('dense_inputs', None) dense_mask = inputs.get('dense_mask', None) dense_type_ids = inputs.get('dense_type_ids', None) else: raise ValueError('Unexpected inputs type to %s.' % self.__class__) if word_embeddings is None: word_embeddings = self._embedding_layer(word_ids) if dense_inputs is not None: # Allow concatenation of the dense embeddings at sequence end if requested # and `unpool_length`` is set as zero if self._append_dense_inputs: if self._unpool_length != 0: raise ValueError( 'unpool_length is not supported by append_dense_inputs now.' ) word_embeddings = tf.concat([word_embeddings, dense_inputs], axis=1) type_ids = tf.concat([type_ids, dense_type_ids], axis=1) mask = tf.concat([mask, dense_mask], axis=1) else: # Concat the dense embeddings at sequence begin so unpool_len can # control embedding not being pooled. word_embeddings = tf.concat([dense_inputs, word_embeddings], axis=1) type_ids = tf.concat([dense_type_ids, type_ids], axis=1) mask = tf.concat([dense_mask, mask], axis=1) # absolute position embeddings position_embeddings = self._position_embedding_layer(word_embeddings) type_embeddings = self._type_embedding_layer(type_ids) embeddings = tf_keras.layers.add( [word_embeddings, position_embeddings, type_embeddings]) embeddings = self._embedding_norm_layer(embeddings) embeddings = self._embedding_dropout(embeddings) if self._embedding_projection is not None: embeddings = self._embedding_projection(embeddings) attention_mask = self._attention_mask_layer(embeddings, mask) encoder_outputs = [] x = embeddings # TODO(b/195972228): attention_mask can be co-generated with pooling. if self._pool_type in (_MAX, _AVG): attention_mask = _pool_and_concat( attention_mask, unpool_length=self._unpool_length, strides=self._pool_strides[0], axes=[1]) for i, layer in enumerate(self._transformer_layers): transformer_output_range = None if i == self._num_layers - 1: transformer_output_range = output_range # Bypass no pooling cases. if self._pool_strides[i] == 1: x = layer( [x, x, attention_mask], output_range=transformer_output_range ) else: # Pools layer for compressing the query length. pooled_inputs = self._att_input_pool_layers[i]( x[:, self._unpool_length:, :]) query_inputs = tf.concat( values=(tf.cast( x[:, :self._unpool_length, :], dtype=pooled_inputs.dtype), pooled_inputs), axis=1) x = layer([query_inputs, x, attention_mask], output_range=transformer_output_range) # Pools the corresponding attention_mask. if i < len(self._transformer_layers) - 1: attention_mask = _pool_and_concat( attention_mask, unpool_length=self._unpool_length, strides=[self._pool_strides[i + 1], self._pool_strides[i]], axes=[1, 2]) encoder_outputs.append(x) elif self._pool_type == _TRUNCATED_AVG: # Compute the attention masks and pooling transforms. # Note we do not compute this in __init__ due to inference converter issue # b/215659399. pooling_transforms = _create_truncated_avg_transforms( self._max_sequence_length, self._pool_strides) attention_masks = _create_truncated_avg_masks(mask, self._pool_strides, pooling_transforms) for i, layer in enumerate(self._transformer_layers): attention_mask = attention_masks[i] transformer_output_range = None if i == self._num_layers - 1: transformer_output_range = output_range # Bypass no pooling cases. if self._pool_strides[i] == 1: x = layer([x, x, attention_mask], output_range=transformer_output_range) else: pooled_inputs = tf.einsum( 'BFD,FT->BTD', tf.cast(x[:, self._unpool_length:, :], _get_policy_dtype() ), # extra casting for faster mixed computation. pooling_transforms[i]) query_inputs = tf.concat( values=(tf.cast( x[:, :self._unpool_length, :], dtype=pooled_inputs.dtype), pooled_inputs), axis=1) x = layer([query_inputs, x, attention_mask], output_range=transformer_output_range) encoder_outputs.append(x) last_encoder_output = encoder_outputs[-1] first_token_tensor = last_encoder_output[:, 0, :] pooled_output = self._pooler_layer(first_token_tensor) return dict( word_embeddings=word_embeddings, embedding_output=embeddings, sequence_output=encoder_outputs[-1], pooled_output=pooled_output, encoder_outputs=encoder_outputs) def get_embedding_table(self): return self._embedding_layer.embeddings def get_embedding_layer(self): return self._embedding_layer def get_config(self): return dict(self._config) @property def transformer_layers(self): """List of Transformer layers in the encoder.""" return self._transformer_layers @property def pooler_layer(self): """The pooler dense layer after the transformer layers.""" return self._pooler_layer @classmethod def from_config(cls, config, custom_objects=None): if 'embedding_layer' in config and config['embedding_layer'] is not None: warn_string = ( 'You are reloading a model that was saved with a ' 'potentially-shared embedding layer object. If you contine to ' 'train this model, the embedding layer will no longer be shared. ' 'To work around this, load the model outside of the Keras API.') print('WARNING: ' + warn_string) logging.warn(warn_string) return cls(**config)