apps/article.py

import streamlit as st
from apps.utils import read_markdown
from .streamlit_tensorboard import st_tensorboard, kill_tensorboard
from .utils import Toc
def app(state=None):
    #kill_tensorboard()
    toc = Toc()
    st.info("Welcome to our Multilingual-VQA demo. Please use the navigation sidebar to move to our demo, or scroll below to read all about our project. 🤗 In case the sidebar isn't properly rendered, please change to a smaller window size and back to full screen.")
    
    st.header("Table of contents")
    toc.placeholder()
    
    toc.header("Introduction and Motivation")
    st.write(read_markdown("intro/intro.md"))
    toc.subheader("Novel Contributions")
    st.write(read_markdown("intro/contributions.md"))
    
    toc.header("Methodology")

    toc.subheader("Pre-training")
    st.write(read_markdown("pretraining/intro.md"))
    # col1, col2 = st.beta_columns([5,5])
    st.image(
        "./misc/article/Multilingual-VQA.png",
        caption="Masked LM model for Image-text Pre-training.",
    )
    toc.subsubheader("MLM Dataset")
    st.write(read_markdown("pretraining/data.md"))
    toc.subsubheader("MLM Model")
    st.write(read_markdown("pretraining/model.md"))
    toc.subsubheader("MLM Training Logs")
    st.info("In case the TensorBoard logs are not displayed, please visit this link: https://huggingface.co/flax-community/multilingual-vqa-pt-ckpts/tensorboard")
    st_tensorboard(logdir='./logs/pretrain_logs', port=6006)
    
    
    toc.subheader("Finetuning")
    toc.subsubheader("VQA Dataset")
    st.write(read_markdown("finetuning/data.md"))
    toc.subsubheader("VQA Model")
    st.write(read_markdown("finetuning/model.md"))
    toc.subsubheader("VQA Training Logs")
    st.info("In case the TensorBoard logs are not displayed, please visit this link: https://huggingface.co/flax-community/multilingual-vqa-pt-60k-ft/tensorboard")
    st_tensorboard(logdir='./logs/finetune_logs', port=6007)
    
    toc.header("Challenges and Technical Difficulties")
    st.write(read_markdown("challenges.md"))
    
    toc.header("Limitations")
    st.write(read_markdown("limitations.md"))
    
    toc.header("Conclusion, Future Work, and Social Impact")
    toc.subheader("Conclusion")
    st.write(read_markdown("conclusion_future_work/conclusion.md"))
    toc.subheader("Future Work")
    st.write(read_markdown("conclusion_future_work/future_work.md"))
    toc.subheader("Social Impact")
    st.write(read_markdown("conclusion_future_work/social_impact.md"))
    
    toc.header("References")
    st.write(read_markdown("references.md"))

    toc.header("Checkpoints")
    st.write(read_markdown("checkpoints/checkpoints.md"))
    toc.subheader("Other Checkpoints")
    st.write(read_markdown("checkpoints/other_checkpoints.md"))
    
    toc.header("Acknowledgements")
    st.write(read_markdown("acknowledgements.md"))

    toc.header("VQA Examples")
    toc.subheader("Color Questions")
    col1, col2, col3 = st.beta_columns([1,1,1])

    col1.image("./sections/examples/men_riding_horses.jpeg", use_column_width="auto", width=300)
    col1.write("**Custom Question**: What color are the horses?")
    col1.write("**Predicted Answer**: brown✅")

    col2.image("./sections/examples/cat_color.jpeg", use_column_width="auto", width=300)
    col2.write("**Custom Question**: What color is the cat?")
    col2.write("**Predicted Answer**: white✅")

    col3.image("./sections/examples/men_happy.jpeg", use_column_width="auto", width=300)
    col3.write("**Custom Question**: What color is the man's jacket?")
    col3.write("**Predicted Answer**: black⚫")

    col1.image("./sections/examples/car_color.jpeg", use_column_width="auto", width=300)
    col1.write("**Actual Question**: What color is the car?")
    col1.write("**Predicted Answer**: blue❎")

    col2.image("./sections/examples/coat_color.jpeg", use_column_width="auto", width=300)
    col2.write("**Actual Question**: What color is this person's coat?")
    col2.write("**Predicted Answer**: blue✅")

    toc.subheader("Counting Questions")

    col1, col2, col3 = st.beta_columns([1,1, 1])

    col1.image("./sections/examples/giraffe_zebra.jpeg", use_column_width="auto", width=300)
    col1.write("**Actual Question**: How many zebras are there?")
    col1.write("**Predicted Answer**: 0❎")

    col2.image("./sections/examples/giraffe_zebra.jpeg", use_column_width="auto", width=300)
    col2.write("**Custom Question**: How many giraffes are there?")
    col2.write("**Predicted Answer**: 2❎")

    col3.image("./sections/examples/teddy.jpeg", use_column_width="auto", width=300)
    col3.write("**Custom Question**: How many teddy bears are present in the image?")
    col3.write("**Predicted Answer**: 3✅")

    col1.image("./sections/examples/candle_count.jpeg", use_column_width="auto", width=300)
    col1.write("**Actual Question**: ¿Cuantas velas hay en el cupcake?")
    col1.write("**English Translation**: How many candles are in the cupcake?")
    col1.write("**Predicted Answer**: 0❎")

    col1.image("./sections/examples/people_picture.jpeg", use_column_width="auto", width=300)
    col1.write("**Actual Question**: ¿A cuánta gente le están tomando una foto?")
    col1.write("**English Translation**: How many people are you taking a picture of?")
    col1.write("**Predicted Answer**: 10❎")

    toc.subheader("Size/Shape Questions")
    col1, col2, col3 = st.beta_columns([1,1,1])
    col1.image("./sections/examples/vase.jpeg", use_column_width="auto", width=300)
    col1.write("**Actual Question**:  What shape is the vase? ")
    col1.write("**Predicted Answer**: round✅")
    

    toc.subheader("Yes/No Questions")
    col1, col2, col3 = st.beta_columns([1,1,1])

    col1.image("./sections/examples/teddy.jpeg", use_column_width="auto", width=300)
    col1.write("**Actual Question**: Sind das drei Teddybären?")
    col1.write("**English Translation**: Are those teddy bears?")
    col1.write("**Predicted Answer**: Ja (yes)✅")

    col2.image("./sections/examples/winter.jpeg", use_column_width="auto", width=300)
    col2.write("**Actual Question**: ¿Se lo tomaron en invierno?")
    col2.write("**English Translation**: Did they take it in winter?")
    col2.write("**Predicted Answer**: si (yes)✅")

    col3.image("./sections/examples/clock.jpeg", use_column_width="auto", width=300)
    col3.write("**Actual Question**: Is the clock ornate? ")
    col3.write("**Predicted Answer**: yes✅")

    col1.image("./sections/examples/decorated_building.jpeg", use_column_width="auto", width=300)
    col1.write("**Actual Question**: Ist das Gebäude orniert?")
    col1.write("**English Translation**: Is the building decorated?")
    col1.write("**Predicted Answer**: Ja (yes)✅")
    
    col2.image("./sections/examples/commuter_train.jpeg", use_column_width="auto", width=300)
    col2.write("**Actual Question**: Ist das ein Pendler-Zug?")
    col2.write("**English Translation**: Is that a commuter train?")
    col2.write("**Predicted Answer**: Ja (yes)❎")

    col3.image("./sections/examples/is_in_a_restaurant.jpeg", use_column_width="auto", width=300)
    col3.write("**Actual Question**: Elle est dans un restaurant?")
    col3.write("**English Translation**: Is she in a restaurant?")
    col3.write("**Predicted Answer**: Oui (yes)❎")

    col1.image("./sections/examples/giraffe_eyes.jpeg", use_column_width="auto", width=300)
    col1.write("**Actual Question**: Est-ce que l'œil de la girafe est fermé?")
    col1.write("**English Translation**: Are the giraffe's eyes closed?")
    col1.write("**Predicted Answer**: Oui (yes)❎")

    toc.subheader("Negatives Test")
    col1, col2, col3 = st.beta_columns([1,1,1])
    col1.image("./sections/examples/men_happy.jpeg", use_column_width="auto", width=300)

    col2.write("**Actual Question**: Is the man happy?")
    col2.write("**Predicted Answer**: Yes✅")

    col3.write("**Actual Question**: Is the man not happy?")
    col3.write("**Predicted Answer**: Yes❎")

    col2.write("**Actual Question**: Is the man sad?")
    col2.write("**Predicted Answer**: No✅")

    col3.write("**Actual Question**: Is the man not sad?")
    col3.write("**Predicted Answer**: No❎")

    col2.write("**Actual Question**: Is the man unhappy?")
    col2.write("**Predicted Answer**: No✅")

    col3.write("**Actual Question**: Is the man not unhappy?")
    col3.write("**Predicted Answer**: No❎")

    toc.subheader("Multilinguality Test")

    toc.subsubheader("Color Question")
    col1, col2, col3 = st.beta_columns([1,1,1])
    col1.image("./sections/examples/truck_color.jpeg", use_column_width="auto", width=300)

    col2.write("**Actual Question**: What color is the building?")
    col2.write("**Predicted Answer**: red✅")

    col3.write("**Actual Question**: Welche Farbe hat das Gebäude?")
    col3.write("**English Translation**: What color is the building?")
    col3.write("**Predicted Answer**: rot (red)✅")

    col2.write("**Actual Question**: ¿De qué color es el edificio?")
    col2.write("**English Translation**: What color is the building?")
    col2.write("**Predicted Answer**: rojo (red)✅")

    col3.write("**Actual Question**: De quelle couleur est le bâtiment ?")
    col3.write("**English Translation**: What color is the building?")
    col3.write("**Predicted Answer**: rouge (red)✅")

    toc.subsubheader("Counting Question")
    col1, col2, col3 = st.beta_columns([1,1,1])
    col1.image("./sections/examples/bear.jpeg", use_column_width="auto", width=300)

    col2.write("**Actual Question**: How many bears do you see?")
    col2.write("**Predicted Answer**: 1✅")

    col3.write("**Actual Question**: Wie viele Bären siehst du?")
    col3.write("**English Translation**: How many bears do you see?")
    col3.write("**Predicted Answer**: 1✅")

    col2.write("**Actual Question**: ¿Cuántos osos ves?")
    col2.write("**English Translation**: How many bears do you see?")
    col2.write("**Predicted Answer**: 1✅")

    col3.write("**Actual Question**: Combien d'ours voyez-vous ?")
    col3.write("**English Translation**: How many bears do you see?")
    col3.write("**Predicted Answer**: 1✅")

    toc.subsubheader("Misc Question")
    col1, col2, col3 = st.beta_columns([1,1,1])
    col1.image("./sections/examples/bench.jpeg", use_column_width="auto", width=300)

    col2.write("**Actual Question**: Where is the bench?")
    col2.write("**Predicted Answer**: field✅")

    col3.write("**Actual Question**: Où est le banc ?")
    col3.write("**English Translation**: Where is the bench?")
    col3.write("**Predicted Answer**: domaine (field)✅")

    col2.write("**Actual Question**: ¿Dónde está el banco?")
    col2.write("**English Translation**: Where is the bench?")
    col2.write("**Predicted Answer**: campo (field)✅")

    col3.write("**Actual Question**: Wo ist die Bank?")
    col3.write("**English Translation**: Where is the bench?")
    col3.write("**Predicted Answer**: Feld (field)✅")


    toc.subheader("Misc Questions")
    col1, col2, col3 = st.beta_columns([1,1,1])

    col1.image("./sections/examples/tennis.jpeg", use_column_width="auto", width=300)
    col1.write("**Actual Question**: ¿Qué clase de juego está viendo la multitud?")
    col1.write("**English Translation**: What kind of game is the crowd watching?")
    col1.write("**Predicted Answer**: tenis (tennis)✅")

    col2.image("./sections/examples/men_body_suits.jpeg", use_column_width="auto", width=300)
    col2.write("**Custom Question**: What are the men wearing?")
    col2.write("**Predicted Answer**: wetsuits✅")

    col3.image("./sections/examples/bathroom.jpeg", use_column_width="auto", width=300)
    col3.write("**Actual Question**: ¿A qué habitación perteneces?")
    col3.write("**English Translation**: What room do you belong to?")
    col3.write("**Predicted Answer**: bano (bathroom)✅")

    col1.image("./sections/examples/men_riding_horses.jpeg", use_column_width="auto", width=300)
    col1.write("**Custom Question**: What are the men riding?")
    col1.write("**Predicted Answer**: horses✅")

    col2.image("./sections/examples/inside_outside.jpeg", use_column_width="auto", width=300)
    col2.write("**Actual Question**: Was this taken inside or outside?")
    col2.write("**Predicted Answer**: inside✅")

    col3.image("./sections/examples/dog_looking_at.jpeg", use_column_width="auto", width=300)
    col3.write("**Actual Question**: Was guckt der Hund denn so?")
    col3.write("**English Translation**: What is the dog looking at?")
    col3.write("**Predicted Answer**: Frisbeescheibe (frisbee)❎")

    toc.generate()