views/record_sets.py

import numpy as np
import pandas as pd
from rdflib import term
import streamlit as st

from core.state import Field
from core.state import Metadata
from core.state import RecordSet
from core.state import SelectedRecordSet
from events.record_sets import handle_record_set_change
from events.record_sets import RecordSetEvent
import mlcroissant as mlc
from utils import needed_field
from views.source import FieldEvent
from views.source import handle_field_change
from views.source import render_references
from views.source import render_source

DATA_TYPES = [
    mlc.DataType.TEXT,
    mlc.DataType.FLOAT,
    mlc.DataType.INTEGER,
    mlc.DataType.BOOL,
    mlc.DataType.URL,
]


def _handle_close_fields():
    st.session_state[SelectedRecordSet] = None


def _handle_on_click_field(
    record_set_key: int,
    record_set: RecordSet,
):
    st.session_state[SelectedRecordSet] = SelectedRecordSet(
        record_set_key=record_set_key,
        record_set=record_set,
    )


def _data_editor_key(record_set_key: int, record_set: RecordSet) -> str:
    return f"{record_set_key}-{record_set.name}-dataframe"


def _get_possible_sources(metadata: Metadata) -> list[str]:
    possible_sources: list[str] = []
    for resource in metadata.distribution:
        possible_sources.append(resource.name)
    for record_set in metadata.record_sets:
        for field in record_set.fields:
            possible_sources.append(f"{record_set.name}/{field.name}")
    return possible_sources


LeftOrRight = tuple[str, str]
Join = tuple[LeftOrRight, LeftOrRight]


def _find_left_or_right(source: mlc.Source) -> LeftOrRight:
    uid = source.uid
    if "/" in uid:
        parts = uid.split("/")
        return (parts[0], parts[1])
    elif source.extract.column:
        return (uid, source.extract.column)
    elif source.extract.json_path:
        return (uid, source.extract.json_path)
    elif source.extract.file_property:
        return (uid, source.extract.file_property)
    else:
        return (uid, None)


def _find_joins(fields: list[Field]) -> set[Join]:
    """Finds the existing joins in the fields."""
    joins: set[Join] = set()
    for field in fields:
        if field.source and field.references:
            left = _find_left_or_right(field.source)
            right = _find_left_or_right(field.references)
            joins.add((left, right))
    return joins


def _handle_create_record_set():
    metadata: Metadata = st.session_state[Metadata]
    metadata.add_record_set(RecordSet(name="new-record-set", description=""))


def _handle_fields_change(record_set_key: int, record_set: RecordSet):
    data_editor_key = _data_editor_key(record_set_key, record_set)
    result = st.session_state[data_editor_key]
    # `result` has the following structure:
    # ```
    # {'edited_rows': {1: {}}, 'added_rows': [], 'deleted_rows': []}
    # ```
    fields = record_set.fields
    for field_key in result["edited_rows"]:
        field = fields[field_key]
        new_fields = result["edited_rows"][field_key]
        for new_field, new_value in new_fields.items():
            if new_field == FieldDataFrame.NAME:
                field.name = new_value
            elif new_field == FieldDataFrame.DESCRIPTION:
                field.description = new_value
            elif new_field == FieldDataFrame.DATA_TYPE:
                field.data_types = [new_value]
    for added_row in result["added_rows"]:
        field = Field(
            name=added_row.get(FieldDataFrame.NAME),
            description=added_row.get(FieldDataFrame.DESCRIPTION),
            data_types=[added_row.get(FieldDataFrame.DATA_TYPE)],
            source=mlc.Source(
                uid="foo",
                node_type="distribution",
                extract=mlc.Extract(column=""),
            ),
            references=mlc.Source(),
        )
        st.session_state[Metadata].add_field(record_set_key, field)
    for field_key in result["deleted_rows"]:
        st.session_state[Metadata].remove_field(record_set_key, field_key)


class FieldDataFrame:
    """Names of the columns in the pd.DataFrame for `fields`."""

    NAME = "Name"
    DESCRIPTION = "Description"
    DATA_TYPE = "Data type"
    SOURCE_UID = "Source"
    SOURCE_EXTRACT = "Source extract"
    SOURCE_TRANSFORM = "Source transform"
    REFERENCE_UID = "Reference"
    REFERENCE_EXTRACT = "Reference extract"


def render_record_sets():
    col1, col2 = st.columns([1, 1])
    with col1:
        _render_left_panel()
    with col2:
        _render_right_panel()


def _render_left_panel():
    """Left panel: visualization of all RecordSets as expandable forms."""
    distribution = st.session_state[Metadata].distribution
    if not distribution:
        st.markdown("Please add resources first.")
        return
    record_sets = st.session_state[Metadata].record_sets
    record_set: RecordSet
    for record_set_key, record_set in enumerate(record_sets):
        title = f"**{record_set.name or '-'}** ({len(record_set.fields)} fields)"
        prefix = f"record-set-{record_set_key}"
        with st.expander(title, expanded=False):
            col1, col2 = st.columns([1, 3])
            key = f"{prefix}-name"
            col1.text_input(
                needed_field("Name"),
                placeholder="Name without special character.",
                key=key,
                value=record_set.name,
                on_change=handle_record_set_change,
                args=(RecordSetEvent.NAME, record_set, key),
            )
            key = f"{prefix}-description"
            col2.text_input(
                "Description",
                placeholder="Provide a clear description of the RecordSet.",
                key=key,
                value=record_set.description,
                on_change=handle_record_set_change,
                args=(RecordSetEvent.DESCRIPTION, record_set, key),
            )
            key = f"{prefix}-is-enumeration"
            st.checkbox(
                "Whether the RecordSet is an enumeration",
                key=key,
                value=record_set.is_enumeration,
                on_change=handle_record_set_change,
                args=(RecordSetEvent.IS_ENUMERATION, record_set, key),
            )

            joins = _find_joins(record_set.fields)
            has_join = st.checkbox(
                "Whether the RecordSet contains joins. To add a new join, add a"
                f" field with a source in `{record_set.name}` and a reference to"
                " another RecordSet or FileSet/FileObject.",
                key=f"{prefix}-has-joins",
                value=bool(joins),
                disabled=True,
            )
            if has_join:
                for left, right in joins:
                    col1, col2, _, col4, col5 = st.columns([2, 2, 1, 2, 2])
                    col1.text_input(
                        "Left join",
                        disabled=True,
                        value=left[0],
                        key=f"{prefix}-left-join-{left}",
                    )
                    col2.text_input(
                        "Left key",
                        disabled=True,
                        value=left[1],
                        key=f"{prefix}-left-key-{left}",
                    )
                    col4.text_input(
                        "Right join",
                        disabled=True,
                        value=right[0],
                        key=f"{prefix}-right-join-{right}",
                    )
                    col5.text_input(
                        "Right key",
                        disabled=True,
                        value=right[1],
                        key=f"{prefix}-right-key-{right}",
                    )
            names = [field.name for field in record_set.fields]
            descriptions = [field.description for field in record_set.fields]
            # TODO(https://github.com/mlcommons/croissant/issues/350): Allow to display
            # several data types, not only the first.
            data_types = [
                field.data_types[0] if field.data_types else None
                for field in record_set.fields
            ]
            fields = pd.DataFrame(
                {
                    FieldDataFrame.NAME: names,
                    FieldDataFrame.DESCRIPTION: descriptions,
                    FieldDataFrame.DATA_TYPE: data_types,
                },
                dtype=np.str_,
            )
            data_editor_key = _data_editor_key(record_set_key, record_set)
            st.markdown(
                f"{needed_field('Fields')} (add/delete fields by directly editing the"
                " table)"
            )
            st.data_editor(
                fields,
                # There is a bug with `st.data_editor` when the df is empty.
                use_container_width=not fields.empty,
                num_rows="dynamic",
                key=data_editor_key,
                column_config={
                    FieldDataFrame.NAME: st.column_config.TextColumn(
                        FieldDataFrame.NAME,
                        help="Name of the field",
                        required=True,
                    ),
                    FieldDataFrame.DESCRIPTION: st.column_config.TextColumn(
                        FieldDataFrame.DESCRIPTION,
                        help="Description of the field",
                        required=False,
                    ),
                    FieldDataFrame.DATA_TYPE: st.column_config.SelectboxColumn(
                        FieldDataFrame.DATA_TYPE,
                        help="The Croissant type",
                        options=DATA_TYPES,
                        required=True,
                    ),
                },
                on_change=_handle_fields_change,
                args=(record_set_key, record_set),
            )

            st.button(
                "Edit fields details",
                key=f"{prefix}-show-fields",
                on_click=_handle_on_click_field,
                args=(record_set_key, record_set),
            )
    st.button(
        "Create a new RecordSet",
        key=f"create-new-record-set",
        type="primary",
        on_click=_handle_create_record_set,
    )


def _render_right_panel():
    """Right panel: visualization of the clicked Field."""
    metadata: Metadata = st.session_state.get(Metadata)
    selected: SelectedRecordSet = st.session_state.get(SelectedRecordSet)
    if not selected:
        return
    record_set = selected.record_set
    record_set_key = selected.record_set_key
    with st.expander("**Fields**", expanded=True):
        for field_key, field in enumerate(record_set.fields):
            prefix = f"{record_set_key}-{field.name}-{field_key}"
            col1, col2, col3 = st.columns([1, 1, 1])

            key = f"{prefix}-name"
            col1.text_input(
                needed_field("Name"),
                placeholder="Name without special character.",
                key=key,
                value=field.name,
                on_change=handle_field_change,
                args=(FieldEvent.NAME, field, key),
            )
            key = f"{prefix}-description"
            col2.text_input(
                "Description",
                placeholder="Provide a clear description of the RecordSet.",
                key=key,
                on_change=handle_field_change,
                value=field.description,
                args=(FieldEvent.DESCRIPTION, field, key),
            )
            if field.data_types:
                data_type = field.data_types[0]
                if isinstance(data_type, str):
                    data_type = term.URIRef(data_type)
                if data_type in DATA_TYPES:
                    data_type_index = DATA_TYPES.index(data_type)
                else:
                    data_type_index = None
            else:
                data_type_index = None
            key = f"{prefix}-datatypes"
            col3.selectbox(
                needed_field("Data type"),
                index=data_type_index,
                options=DATA_TYPES,
                key=key,
                on_change=handle_field_change,
                args=(FieldEvent.DATA_TYPE, field, key),
            )
            possible_sources = _get_possible_sources(metadata)
            render_source(
                record_set_key, record_set, field, field_key, possible_sources
            )
            render_references(
                record_set_key, record_set, field, field_key, possible_sources
            )

            st.divider()

        st.button(
            "Close",
            key=f"{record_set.name}-{record_set_key}-close-fields",
            type="primary",
            on_click=_handle_close_fields,
        )