| import tensorflow as tf | |
| from tensorflow import keras | |
| from transformer import TransformerEncoder, TransformerDecoderLayer, get_look_ahead_mask, get_padding_mask | |
| class TFPort(keras.Model): | |
| def __init__(self, crop_size, embed_dim, encoder_layers, decoder_layers, input_seq_length, | |
| target_seq_length, hidden_dim, num_heads, input_vocab_size, dropout_rate=0.1): | |
| super().__init__() | |
| self.image_encoder = keras.applications.InceptionV3( | |
| include_top=False, | |
| weights='imagenet', | |
| input_shape=crop_size + (3,), | |
| ) | |
| self.image_encoder.trainable = False | |
| self.conv = keras.layers.Conv2D(embed_dim, 1) | |
| self.ingredient_encoder = TransformerEncoder(encoder_layers, input_seq_length, embed_dim, hidden_dim, | |
| num_heads, input_vocab_size, dropout_rate=dropout_rate) | |
| self.portion_embedding = PortionEmbedding(target_seq_length, embed_dim) | |
| self.dropout = keras.layers.Dropout(dropout_rate) | |
| self.decoder_layers = [ | |
| TransformerDecoderLayer(embed_dim, hidden_dim, num_heads, dropout_rate=dropout_rate) | |
| for _ in range(decoder_layers) | |
| ] | |
| self.linear = keras.layers.Dense(1, activation="relu") | |
| def call(self, inputs, training=False): | |
| image, ingredients, targets = inputs | |
| padding_mask = get_padding_mask(ingredients) | |
| encoded_img = self.image_encoder(image, training=False) | |
| encoded_img = self.conv(encoded_img, training=training) | |
| encoded_img = tf.reshape(encoded_img, (tf.shape(encoded_img)[0], -1, tf.shape(encoded_img)[3])) | |
| encoded_ingr = self.ingredient_encoder(ingredients, padding_mask, training=training) | |
| encoder_outputs = tf.concat([encoded_img, encoded_ingr], axis=1) | |
| img_mask = tf.ones((tf.shape(encoded_img)[0], 1, tf.shape(encoded_img)[1]), dtype=tf.int32) | |
| padding_mask = tf.concat([img_mask, padding_mask], axis=2) | |
| look_ahead_mask = get_look_ahead_mask(targets) | |
| x = self.portion_embedding(targets) | |
| x = self.dropout(x, training=training) | |
| for i in range(len(self.decoder_layers)): | |
| x = self.decoder_layers[i](x, encoder_outputs, look_ahead_mask, training=training, | |
| padding_mask=padding_mask) | |
| x = self.linear(x) | |
| return tf.squeeze(x) | |
| class PortionEmbedding(keras.layers.Layer): | |
| def __init__(self, seq_length, embed_dim): | |
| super().__init__() | |
| self.linear = keras.layers.Dense(embed_dim) | |
| self.position_embeddings = keras.layers.Embedding( | |
| input_dim=seq_length, output_dim=embed_dim | |
| ) | |
| def call(self, inputs): | |
| positions = tf.range(start=0, limit=tf.shape(inputs)[-1], delta=1) | |
| embedded_portions = self.linear(inputs[:, :, tf.newaxis]) | |
| embedded_positions = self.position_embeddings(positions) | |
| return embedded_portions + embedded_positions | |