1st

.gitignore

@@ -0,0 +1,245 @@
+models/
+data/
+**/model_checkpoints
+**/outputs
+training_with_callbacks/
+*.ipynb
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+.venv312
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# MacOS
+.DS_Store
+
+# Frontend
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# misc
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# vercel
+.vercel
+
+# Pycharm
+.idea
+
+# Env vars - to be updated
+/infra/ci/secret.yaml
+
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
+.terraform.lock.hcl
+# ignore .vscode
+.vscode
+
+# Ignore sensitive data - k8s env vars
+infra/environments/chatbot-dev/dev_secret.yaml
+infra/environments/chatbot-prod/prod_secret.yaml
+infra/environments/tt-chatbot-prod/prod_secret.yaml
+
+# yarn file
+yarn.lock
+
+# ignore migrations django
+**/migrations/**
+!**/migrations
+!**/migrations/__init__.py
+
+# Gradio
+**/.gradio
+
+# Lightning
+**/lightning_logs

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+    #-   id: check-added-large-files
+    -   id: fix-byte-order-marker
+    -   id: check-case-conflict
+    -   id: check-json
+    -   id: check-yaml
+        args: ['--unsafe']
+    -   id: detect-aws-credentials
+        args: [--allow-missing-credentials]
+    -   id: detect-private-key
+    -   id: end-of-file-fixer
+    -   id: mixed-line-ending
+    -   id: trailing-whitespace
+-   repo: https://github.com/asottile/add-trailing-comma
+    rev: v3.1.0
+    hooks:
+    -   id: add-trailing-comma
+-   repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+    -   id: isort
+        name: isort (python)
+        args: [--settings-path=pyproject.toml]
+    -   id: isort
+        name: isort (cython)
+        types: [cython]
+    -   id: isort
+        name: isort (pyi)
+        types: [pyi]
+-   repo: https://github.com/psf/black
+    rev: 24.10.0
+    hooks:
+    -   id: black
+        args: [--config=pyproject.toml]
+-   repo: https://github.com/pycqa/flake8.git
+    rev: 6.1.0
+    hooks:
+    -   id: flake8
+        args: [--ignore, "E203, W503", --max-line-length, "79"]
+-   repo: https://github.com/kynan/nbstripout
+    rev: 0.8.1
+    hooks:
+    -   id: nbstripout
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v3.19.0
+    hooks:
+    -   id: pyupgrade
+        args: [--py36-plus]

.pre-commit-setting.toml

@@ -0,0 +1,22 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+[tool.black]
+line-length = 79
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.idea
+  | \.pytest_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''
+
+[flake8]
+ignore = E203, W503
+max-line-length = 79

.sample-env

@@ -0,0 +1,4 @@
+[API_KEY]
+OPENAI_API_KEY=your_api_key  # Replace with your actual OpenAI API key
+GEMINI_API_KEY=your_api_key
+TOGETHER_API_KEY=your_api_key

app.py

@@ -0,0 +1,400 @@
+import warnings
+
+import torchvision.transforms as transforms
+from google_img_source_search import ReverseImageSearcher
+
+# from src.images.CNN_model_classifier import predict_cnn
+# from src.images.diffusion_model_classifier import (
+#     ImageClassifier,
+#     predict_single_image,
+# )
+
+warnings.simplefilter(
+    action="ignore",
+    category=FutureWarning,
+)  # disable FutureWarning
+
+import gradio as gr  # noqa: E402
+from transformers import (  # noqa: E402
+    AutoModelForSequenceClassification,
+    AutoTokenizer,
+    pipeline,
+)
+
+from src.texts.MAGE.deployment import (  # noqa: E402
+    detect,
+    preprocess,
+)
+from src.texts.PASTED.pasted_lexicon import Detector  # noqa: E402
+from src.texts.Search_Text.search import (  # noqa: E402
+    get_important_sentences,
+    get_keywords,
+    is_human_written,
+)
+from src.images.Search_Image.search import (
+    compare_images,
+    get_image_from_path,
+    get_image_from_url,
+)
+
+
+def convert_score_range(score):
+    """
+    Converts a score from the range [0, 1] to [-1, 1].
+
+    Args:
+    score: The original score in the range [0, 1].
+
+    Returns:
+    The converted score in the range [-1, 1].
+    """
+
+    return 2 * score - 1
+
+
+def generate_highlighted_text(text_scores):
+    """
+    Generates a highlighted text string based on the given text and scores.
+
+    Args:
+        text_scores: A list of tuples, where each tuple contains a text
+            segment and its score.
+
+    Returns:
+        A string of HTML code with highlighted text.
+    """
+    highlighted_text = ""
+    for text, score in text_scores:
+        # Map score to a color using a gradient
+        color = f"rgba(255, 0, 0, {1 - score})"  # Red to green gradient
+        highlighted_text += (
+            f"<span style='background-color: {color}'>{text}</span>"  # noqa
+        )
+    return highlighted_text
+
+
+def separate_characters_with_mask(text, mask):
+    """Separates characters in a string and pairs them with a mask sign.
+
+    Args:
+        text: The input string.
+
+    Returns:
+        A list of tuples, where each tuple contains a character and a mask.
+    """
+
+    return [(char, mask) for char in text]
+
+
+def detect_ai_text(model_name, search_engine, text):
+    if search_engine is True:
+        keywords = get_keywords(text)
+        important_sentences = get_important_sentences(text, keywords)
+        predictions = is_human_written(important_sentences[0])
+        print("keywords: ", keywords)
+        print("important_sentences: ", important_sentences)
+        print("predictions: ", predictions)
+        if predictions == -1:
+            caption = "[Found exact match] "
+            text_scores = list(zip([caption, text], [0, predictions]))
+            print("text_scores: ", text_scores)
+            return text_scores
+
+    if model_name == "SimLLM":
+        tokenize_input = SimLLM_tokenizer(text, return_tensors="pt")
+        outputs = SimLLM_model(**tokenize_input)
+        predictions = outputs.logits.argmax(dim=-1).item()
+        if predictions == 0:
+            predictions = "human-written"
+        else:
+            predictions = "machine-generated"
+
+    elif model_name == "MAGE":
+        processed_text = preprocess(text)
+        predictions = detect(
+            processed_text,
+            MAGE_tokenizer,
+            MAGE_model,
+            device,
+        )
+
+    elif model_name == "chatgpt-detector-roberta":
+        predictions = roberta_pipeline_en(text)[0]["label"]
+        if predictions == "Human":
+            predictions = "human-written"
+        else:  # ChatGPT
+            predictions = "machine-generated"
+    elif model_name == "PASTED-Lexical":
+        predictions = detector(text)
+
+    if model_name != "PASTED-Lexical":
+        text_scores = list(zip([text], [predictions]))
+    else:
+        text_scores = []
+        for text, score in predictions:
+            new_score = convert_score_range(score)  # normalize score
+            text_scores.append((text, new_score))
+
+    return text_scores
+
+
+diffusion_model_path = (
+    "src/images/Diffusion/model_checkpoints/"
+    "image-classifier-step=7007-val_loss=0.09.ckpt"
+)
+cnn_model_path = "src/images/CNN/model_checkpoints/blur_jpg_prob0.5.pth"
+
+
+def detect_ai_image(input_image_path, search_engine):
+    # if search_engine is True:
+        # Search image
+        
+        rev_img_searcher = ReverseImageSearcher()
+        search_items = rev_img_searcher.search_by_file(input_image_path)
+        min_result_difference = 5000
+        result_image_url = ""
+        input_image = get_image_from_path(input_image_path)
+        
+        for search_item in search_items:
+            # print(f'Title: {search_item.page_title}')
+            # print(f'Site: {search_item.page_url}')
+            # print(f'Img: {search_item.image_url}\n')
+
+            # Compare each search result image with the input image
+            result_image = get_image_from_url(search_item.image_url)
+            # input_image = get_image_from_url(search_item.image_url)
+            result_difference = compare_images(result_image, input_image)
+            
+            print(f"Difference with search result: {result_difference}")
+            print(f"Result image url: {search_item.page_url}\n")
+
+            if min_result_difference > result_difference:
+                min_result_difference = result_difference
+                result_image_url = search_item.image_url
+                result_page_url = search_item.page_url
+                
+                
+            if result_difference == 0:
+                break
+
+
+        if min_result_difference == 0:
+            result = f"<h1>Input image is LIKELY SIMILAR to image from:</h1>"\
+                    f"<ul>"\
+                    f'<li>\nPage URL: <a href="url">{result_page_url}</a></li>'\
+                    f'<li>\nImage URL: <a href="url">{result_image_url}</a></li>'\
+                    f"<li>\nDifference score: {min_result_difference}</li>"\
+                    f"</ul>"
+        elif 10 > min_result_difference > 0:
+            result = f"<h1>Input image is potentially a VARIATRION from:</h1>"\
+                    f"<ul>"\
+                    f'<li>\nPage URL: <a href="url">{result_page_url}</a></li>'\
+                    f'<li>\nImage URL: <a href="url">{result_image_url}</a></li>'\
+                    f"<li>\nDifference score: {min_result_difference}</li>"\
+                    f"</ul>"
+        elif min_result_difference < 5000:
+            result = f"<h1>Input image is not similar to any search results.</h1>"\
+                    f"<ul>"\
+                    f'<li>\nPage URL: <a href="url">{result_page_url}</a></li>'\
+                    f'<li>\nImage URL: <a href="url">{result_image_url}</a></li>'\
+                    f"<li>\nDifference score: {min_result_difference}</li>"\
+                    f"</ul>"
+        else:
+            result = f"<h1>No search result found.</h1>"\
+        
+        return result
+
+    # def get_prediction_diffusion(image):
+    #     model = ImageClassifier.load_from_checkpoint(diffusion_model_path)
+
+    #     prediction = predict_single_image(image, model)
+    #     return (prediction >= 0.5, prediction)
+
+    # def get_prediction_cnn(image):
+    #     prediction = predict_cnn(image, cnn_model_path)
+    #     return (prediction >= 0.5, prediction)
+
+    # # Define the transformations for the image
+    # transform = transforms.Compose(
+    #     [
+    #         transforms.Resize((224, 224)),  # Image size expected by ResNet50
+    #         transforms.ToTensor(),
+    #         transforms.Normalize(
+    #             mean=[0.485, 0.456, 0.406],
+    #             std=[0.229, 0.224, 0.225],
+    #         ),
+    #     ],
+    # )
+    # image_tensor = transform(inp)
+    # pred_diff, prob_diff = get_prediction_diffusion(image_tensor)
+    # pred_cnn, prob_cnn = get_prediction_cnn(image_tensor)
+    # verdict = (
+    #     "AI Generated" if (pred_diff or pred_cnn) else "No GenAI detected"
+    # )
+    # return (
+    #     f"<h1>{verdict}</h1>"
+    #     f"<ul>"
+    #     f"<li>Diffusion detection score: {prob_diff:.1%} "
+    #     f"{'(MATCH)' if pred_diff else ''}</li>"
+    #     f"<li>CNN detection score: {prob_cnn:.1%} "
+    #     f"{'(MATCH)' if pred_cnn else ''}</li>"
+    #     f"</ul>"
+    # )
+
+
+# Define GPUs
+device = "cpu"  # use 'cuda:0' if GPU is available
+
+# init MAGE
+model_dir = "yaful/MAGE"  # model in huggingface
+MAGE_tokenizer = AutoTokenizer.from_pretrained(model_dir)
+MAGE_model = AutoModelForSequenceClassification.from_pretrained(model_dir).to(
+    device,
+)
+
+# init chatgpt-detector-roberta
+model_dir = "Hello-SimpleAI/chatgpt-detector-roberta"  # model in huggingface
+roberta_pipeline_en = pipeline(task="text-classification", model=model_dir)
+
+# init PASTED
+model_dir = "linzw/PASTED-Lexical"
+detector = Detector(model_dir, device)
+
+# init SimLLM
+model_path = "./models/single_model_detector"
+SimLLM_tokenizer = AutoTokenizer.from_pretrained(model_path)
+SimLLM_model = AutoModelForSequenceClassification.from_pretrained(model_path)
+
+# Init variable for UI
+title = """
+<center>
+
+<h1> AI-generated content detection </h1>
+<b> Demo by NICT & Tokyo Techies <b>
+
+</center>
+"""
+
+examples = [
+    [
+        "SimLLM",
+        False,
+        """\
+The BBC's long-running consumer rights series Watchdog is to end as a \
+standalone programme, instead becoming part of The One Show. Watchdog \
+began in 1980 as a strand of Nationwide, but proved so popular it \
+became a separate programme in 1985. Co-host Steph McGovern has moved \
+to Channel 4, but Matt Allwright and Nikki Fox will stay to front the \
+new strand. The BBC said they would investigate viewer complaints all \
+year round rather than for two series a year.
+""",
+    ],
+    [
+        "chatgpt-detector-roberta",
+        False,
+        """\
+Artificial intelligence (AI) is the science of making machines \
+intelligent. It enables computers to learn from data, recognize \
+patterns, and make decisions. AI powers many technologies we use \
+daily, from voice assistants to self-driving cars. It's rapidly \
+evolving, promising to revolutionize various industries and reshape \
+the future.""",
+    ],
+]
+
+model_remark = """<left>
+Model sources:
+<a href="https://github.com/Tokyo-Techies/prj-nict-ai-content-detection">SimLLM</a>,
+<a href="https://github.com/yafuly/MAGE">MAGE</a>,
+<a href="https://huggingface.co/Hello-SimpleAI/chatgpt-detector-roberta">chatgpt-detector-roberta</a>,
+<a href="https://github.com/Linzwcs/PASTED">PASTED-Lexical</a>.
+</left>
+"""  # noqa: E501
+
+image_samples = [
+    ["src/images/samples/fake_dalle.jpg", "Generated (Dall-E)"],
+    ["src/images/samples/fake_midjourney.png", "Generated (MidJourney)"],
+    ["src/images/samples/fake_stable.jpg", "Generated (Stable Diffusion)"],
+    ["src/images/samples/fake_cnn.png", "Generated (GAN)"],
+    ["src/images/samples/real.png", "Organic"],
+    [
+        "https://p.potaufeu.asahi.com/1831-p/picture/27695628/89644a996fdd0cfc9e06398c64320fbe.jpg",  # noqa E501
+        "Internet GenAI",
+    ],
+]
+image_samples_path = [i[0] for i in image_samples]
+
+# UI
+with gr.Blocks() as demo:
+    with gr.Row():
+        gr.HTML(title)
+    with gr.Row():
+        with gr.Tab("Text"):
+            with gr.Row():
+                with gr.Column():
+                    model = gr.Dropdown(
+                        [
+                            "SimLLM",
+                            "MAGE",
+                            "chatgpt-detector-roberta",
+                            "PASTED-Lexical",
+                        ],
+                        label="Detection model",
+                    )
+                    search_engine = gr.Checkbox(label="Use search engine")
+                    gr.HTML(model_remark)
+                with gr.Column():
+                    text_input = gr.Textbox(
+                        label="Input text",
+                        placeholder="Enter text here...",
+                        lines=5,
+                    )
+
+            output = gr.HighlightedText(
+                label="Detection results",
+                combine_adjacent=True,
+                show_legend=True,
+                color_map={
+                    "human-written": "#7d58cf",
+                    "machine-generated": "#e34242",
+                },
+            )
+
+            gr.Examples(
+                examples=examples,
+                inputs=[model, search_engine, text_input],
+            )
+            model.change(
+                detect_ai_text,
+                inputs=[model, search_engine, text_input],
+                outputs=output,
+            )
+            search_engine.change(
+                detect_ai_text,
+                inputs=[model, search_engine, text_input],
+                outputs=output,
+            )
+            text_input.change(
+                detect_ai_text,
+                inputs=[model, search_engine, text_input],
+                outputs=output,
+            )
+        with gr.Tab("Images"):
+            with gr.Row():
+                input_image = gr.Image(type="filepath")
+                with gr.Column():
+                    output_image = gr.Markdown(height=400)
+            gr.Examples(
+                examples=image_samples,
+                inputs=input_image,
+            )
+
+            input_image.change(
+                detect_ai_image,
+                inputs=input_image,
+                outputs=output_image,
+            )
+
+
+# demo.launch(share=True)
+demo.launch(allowed_paths=image_samples_path, share=True)

application.py

@@ -0,0 +1,137 @@
+import os
+
+import gradio as gr
+import openai
+import requests
+from PIL import Image
+import re
+
+from src.application.url_reader import URLReader
+
+OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
+openai.api_key = os.getenv('OPENAI_API_KEY')
+GOOGLE_API_KEY = os.getenv('GOOGLE_API_KEY')
+SEARCH_ENGINE_ID = os.getenv('SEARCH_ENGINE_ID')
+
+def load_url(url):
+    """
+    Load content from the given URL.
+    """
+    content = URLReader(url)
+    image = None
+    header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36'}
+    try:
+        response = requests.get(
+            url, 
+            headers = header,
+            stream = True
+        )
+        response.raise_for_status()  # Raise an exception for bad status codes
+        
+        image_response = requests.get(content.top_image, stream=True)
+        try:
+            image = Image.open(image_response.raw)
+        except:
+            print(f"Error loading image from {content.top_image}")
+            
+    except (requests.exceptions.RequestException, FileNotFoundError) as e:
+        print(f"Error fetching image: {e}")
+
+    return content.title, content.text, image
+
+
+def replace_terms(text, input_term, destination_term):
+    # Replace input_term with destination_term in the text
+    modified_text = re.sub(input_term, destination_term, text) 
+    return modified_text
+
+def generate_content(model1, model2, title, content):
+    # Generate text using the selected models
+    full_content = ""
+    input_type = ""
+    if title and content:
+        full_content = title + "\n" + content
+        input_type = "title and content"
+    elif title:
+        full_content = title
+        input_type = "title"
+    elif content:
+        full_content = title
+        input_type = "content"
+        
+def generate_text(model, full_context, input_type):
+    # Generate text using the selected model
+    if input_type == "":
+        prompt = "Generate a random fake news article"
+    else:
+        prompt = f"Generate a fake news article (title and content) based on the following {input_type}: {full_context}"
+        
+    try:
+        response = openai.ChatCompletion.create(
+        model=model, 
+        messages=[
+            {"role": "user", "content": prompt}
+        ]
+        )
+        return response.choices[0].message.content
+
+    except openai.error.OpenAIError as e:
+        print(f"Error interacting with OpenAI API: {e}")
+        return "An error occurred while processing your request."
+
+# Define the GUI
+with gr.Blocks() as demo:
+    gr.Markdown("# Fake News Detection")
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown("## Settings")
+            gr.Markdown("This tool generates fake news by modifying the content of a given URL.")
+
+            with gr.Accordion("1. Enter a URL"):
+                #gr.Markdown("  1. Enter a URL.")
+                url_input = gr.Textbox(
+                    label="URL",
+                    value="https://bbc.com/future/article/20250110-how-often-you-should-wash-your-towels-according-to-science",
+                    )
+                load_button = gr.Button("Load an URL...")
+                
+            with gr.Accordion("2. Select a content-generation model", open=True):
+                with gr.Row():
+                    model1_dropdown = gr.Dropdown(choices=["GPT 4o", "GPT 4o-mini"], label="Text-generation model")
+                    model2_dropdown = gr.Dropdown(choices=["Dall-e", "Stable Diffusion"], label="Image-generation model")
+                generate_button = gr.Button("Random generation...")
+
+            with gr.Accordion("3. Replace any terms", open=True):
+                with gr.Row():
+                    input_term_box = gr.Textbox(label="Input Term")
+                    destination_term_box = gr.Textbox(label="Destination Term")
+                replace_button = gr.Button("Replace term...")
+            
+            process_button = gr.Button("Process")
+
+        with gr.Column(scale=2):
+            gr.Markdown("## News contents")
+            title_input = gr.Textbox(label="Title", value="")
+            with gr.Row():
+                image_view = gr.Image(label="Image") 
+                content_input = gr.Textbox(label="Content", value="", lines=15)
+
+    
+
+    # Connect events
+    load_button.click(
+        load_url, 
+        inputs=url_input, 
+        outputs=[title_input, content_input, image_view]
+        )
+    replace_button.click(replace_terms, 
+                        inputs=[content_input, input_term_box, destination_term_box], 
+                        outputs=content_input)
+    process_button.click(generate_text, 
+                        inputs=[url_input, model1_dropdown, model2_dropdown, input_term_box, destination_term_box, title_input, content_input], 
+                        outputs=[title_input, content_input])
+
+    #url_input.change(load_image, inputs=url_input, outputs=image_view) 
+
+demo.launch()

demo.py

@@ -0,0 +1,309 @@
+import os
+
+from src.images.Search_Image.search import find_similar_img_from_url
+
+import re
+
+import gradio as gr
+
+from src.images.Search_Image.image_model_share import (
+    image_generation_detection,
+)
+from src.texts.Search_Text._text_detection_share import (
+    UNKNOWN,
+    abstract_detect_generated_text,
+)
+from src.texts.Search_Text.fake_text_generation_share import (
+    highlight_overlap_by_word_to_list,
+)
+
+os.environ["no_proxy"] = "localhost,127.0.0.1,::1"
+
+TEMP_IMAGE = "temp_image.jpg"
+TEMP_INPUT_IMAGE = "temp_input_image.jpg"
+
+HUMAN_IMAGE = "data/test_data/human_news.jpg"
+
+HUMAN_CAPTION = "Stoke City have secured West Brom striker Saido Berahino for £12 million on a five-and-a-half-year contract."
+HUMAN_CONTENT = """
+Tracey Jolliffe has already donated a kidney, 16 eggs and 80 pints of blood, and intends to leave her brain to science. She is now hoping to give away part of her liver to a person she may never meet.
+"If I had another spare kidney, I'd do it again," Tracey tells the BBC's Victoria Derbyshire programme.
+She is what is known as an "altruistic donor" - someone willing to give away an organ to potentially help save the life of a complete stranger.
+A microbiologist in the NHS, and the daughter of two nurses, she has spent her life learning about the importance of healthcare from a professional standpoint.
+But she has also been keen to make a difference on a personal level.
+"I signed up to donate blood, and to the bone marrow register, when I was 18," she says.
+Now 50, her wish to donate has become gradually more expansive.
+In 2012, she was one of fewer than 100 people that year to donate a kidney without knowing the recipient's identity - and now supports the charity Give A Kidney, encouraging others to do the same.
+As of 30 September 2016, 5,126 people remain on the NHS kidney transplant waiting list.
+Tracey's kidney donation, in all likelihood, will have saved someone's life.
+"I remind myself of it every day when I wake up," she says, rightly proud of her life-changing actions.
+It was not, however, a decision taken on the spur of a moment.
+Donating a kidney is an "involved process", she says, with suitability assessments taking at least three months to complete.
+Tests leading up to the transplant include X-rays, heart tracing and a special test of kidney function, which involves an injection and a series of blood tests.
+"It is not something to do if you're scared of needles," she jokes.
+The risks associated with donating, however, are relatively low for those deemed healthy enough to proceed, with a mortality rate of about one in 3,000 - roughly the same as having an appendix removed.
+Compared with the general public, NHS Blood and Transplant says, most kidney donors have equivalent - or better - life expectancy than the average person.
+Tracey says she was in hospital for five days after her operation but felt "back to normal" within six weeks.
+"""
+
+HUMAN_NEWS_CNN = """
+Mayotte authorities fear hunger and disease after cyclone, as death toll rises in Mozambique
+Cyclone Chido caused devastation in Mayotte and authorities are now rushing to prevent disease and hunger spreading in the French overseas territory Sipa USA
+Authorities in Mayotte were racing on Tuesday to stop hunger, disease and lawlessness from spreading in the French overseas territory after the weekend’s devastating cyclone, while Mozambique reported dozens of deaths from the storm.
+Hundreds or even thousands could be dead in Mayotte, which took the strongest hit from Cyclone Chido, French officials have said. The storm laid waste to large parts of the archipelago off east Africa, France’s poorest overseas territory, before striking continental Africa.
+With many parts of Mayotte still inaccessible and some victims buried before their deaths could be officially counted, it may take days to discover the full extent of the destruction.
+So far, 22 deaths and more than 1,400 injuries have been confirmed, Ambdilwahedou Soumaila, the mayor of the capital Mamoudzou, told Radio France Internationale on Tuesday morning.
+“The priority today is water and food,” Soumaila said. “There are people who have unfortunately died where the bodies are starting to decompose that can create a sanitary problem.”
+“We don’t have electricity. When night falls, there are people who take advantage of that situation.”
+
+Rescue workers operate in storm-hit Mayotte on Wednesday.
+Rescue workers operate in storm-hit Mayotte on Wednesday. Securite Civile via Reuters
+Twenty tonnes of food and water are due to start arriving on Tuesday by air and sea. The French government said late on Monday it expects 50% of water supplies to be restored within 48 hours and 95% within the week.
+France’s interior ministry announced that a curfew would go into effect on Tuesday night from 10 p.m. to 4 a.m. local time.
+Rescue workers have been searching for survivors amid the debris of shantytowns bowled over by 200 kph (124 mph) winds.
+Chido was the strongest storm to strike Mayotte in more than 90 years, French weather service Meteo France said. In Mozambique, it killed at least 34 people, officials said on Tuesday. Another seven died in Malawi.
+Drone footage from Mozambique’s Cabo Delgado province, already experiencing a humanitarian crisis due to an Islamist insurgency, showed razed thatched-roof houses near the beach and personal belongings scattered under the few palm trees still standing.
+
+Dispute over immigration
+French President Emmanuel Macron said after an emergency cabinet meeting on Monday that he would visit Mayotte in the coming days, as the disaster quickly fueled a political back-and-forth about immigration, the environment and France’s treatment of its overseas territories.
+Mayotte has been grappling with unrest in recent years, with many residents angry at illegal immigration and inflation.
+More than three-quarters of its roughly 321,000 people live in relative poverty, and about one-third are estimated to be undocumented migrants, most from nearby Comoros and Madagascar.
+The territory has become a stronghold for the far-right National Rally with 60% voting for Marine Le Pen in the 2022 presidential election runoff.
+France’s acting Interior Minister Bruno Retailleau, from the conservative Republicans party, told a news conference in Mayotte that the early warning system had worked “perfectly” but many of the undocumented had not come to designated shelters.
+People stand amid uprooted trees and debris after cyclone Chido hit Mecufi district, Cabo Delgado province, Mozambique, on December 16.
+People stand amid uprooted trees and debris after cyclone Chido hit Mecufi district, Cabo Delgado province, Mozambique, on December 16. UNICEF Mozambique via Reuters
+Other officials have said undocumented migrants may have been afraid to go to shelters for fear of being arrested.
+The toll of the cyclone, Retailleau said in a later post on X, underscored the need to address “the migration question.”
+“Mayotte is the symbol of the drift that (French) governments have allowed to take hold on this issue,” he said. “We will need to legislate so that in Mayotte, like everywhere else on the national territory, France retakes control of its immigration.”
+Left-wing politicians, however, have pointed the finger at what they say is the government’s neglect of Mayotte and failure to prepare for natural disasters linked to climate change.
+Socialist Party chairman Olivier Faure blasted Retailleau’s comments in an X post.
+“He could have interrogated the role of climate change in producing more and more intense climate disasters. He could have rallied against the extreme poverty that makes people more vulnerable to cyclones,” said Faure.
+“No, he has resumed his crusade against migrants.”
+Prime Minister Francois Bayrou, appointed last week to steer France out of a political crisis, faced criticism after he went to the town of Pau, where he is the mayor, to attend a municipal council meeting on Monday, instead of visiting Mayotte.
+"""
+
+HUMAN_NEWS_CNN_IMAGE = "human_cnn.webp"
+# generate a short news related to sport
+
+# opposite
+OPPOSITE_NEWS = """
+Tracey Jolliffe has never donated a kidney, any eggs, or blood, and has no plans to leave her brain to science. She is not considering giving away any part of her liver to someone she knows.
+"If I had another spare kidney, I wouldn't do it again," Tracey tells the BBC's Victoria Derbyshire programme.
+She is not an "altruistic donor" - someone unwilling to give away an organ to potentially save the life of a complete stranger.
+A microbiologist outside the NHS, with parents who were not in healthcare, she has spent her life without focusing on the importance of healthcare from a professional standpoint.
+She has also not been eager to make a difference on a personal level.
+"I never signed up to donate blood, nor to the bone marrow register, when I was 18," she says.
+Now 50, her interest in donating has not expanded.
+In 2012, she was not among the few people that year to donate a kidney without knowing the recipient's identity - and does not support the charity Give A Kidney, discouraging others from doing the same.
+As of 30 September 2016, 5,126 people remain on the NHS kidney transplant waiting list.
+Tracey's decision not to donate a kidney hasn't saved anyone's life.
+"I never think about it when I wake up," she says, indifferent about her choices.
+It was not a decision made after careful consideration.
+Donating a kidney is not an "involved process", she says, with suitability assessments taking less than three months to complete.
+Tests leading up to the transplant do not include X-rays, heart tracing, or a special test of kidney function, which does not involve an injection or any blood tests.
+"It is something to do if you're scared of needles," she jokes.
+The risks associated with donating, however, are relatively high for those not deemed healthy enough to proceed, with a high mortality rate - much greater than having an appendix removed.
+Compared with the general public, NHS Blood and Transplant says, most kidney donors have worse life expectancy than the average person.
+Tracey says she was not in hospital after any operation and did not feel "back to normal" within six weeks.
+"""
+
+PARAPHASE_NEWS = """
+Tracey Jolliffe has generously donated a kidney, 16 eggs, and 80 pints of blood, and plans to donate her brain to science. She now hopes to donate part of her liver to someone she may never meet. "If I had another spare kidney, I'd do it again," she shares with the BBC's Victoria Derbyshire program. Known as an "altruistic donor," Tracey is willing to donate organs to help save the lives of strangers.
+As a microbiologist in the NHS and the daughter of two nurses, Tracey has always understood the importance of healthcare professionally. However, she also strives to make a personal impact. "I signed up to donate blood and joined the bone marrow register at 18," she explains. Now 50, her desire to donate has expanded over the years.
+In 2012, Tracey was among fewer than 100 people that year who donated a kidney without knowing the recipient. She now supports Give A Kidney, a charity that encourages others to donate. As of 30 September 2016, 5,126 people were on the NHS kidney transplant waiting list. Tracey's kidney donation likely saved a life. "I remind myself of it every day when I wake up," she says, proud of her life-changing decision.
+Donating a kidney was not a spontaneous decision for Tracey. It is a complex process, she explains, with suitability assessments taking at least three months. Pre-transplant tests include X-rays, heart monitoring, and a special kidney function test involving an injection and multiple blood tests. "It's not for those afraid of needles," she jokes.
+For healthy individuals, the risks of donating a kidney are relatively low, with a mortality rate of about one in 3,000, similar to having an appendix removed. According to NHS Blood and Transplant, most kidney donors have the same or better life expectancy compared to the general population. Tracey was hospitalized for five days after her operation and felt "back to normal" within six weeks.
+"""
+
+MACHINE_IMAGE = "data/test_data/machine_news.png"
+# MACHINE_CAPTION = "Argentina Secures Victory in Thrilling Friendly Match Against Brazil"
+MACHINE_CONTENT = """
+Tracey Jolliffe has already donated a kidney, 16 eggs, and 80 pints of blood, and she intends to leave her brain to science. She is now hoping to give away part of her liver to a person she may never meet.
+"If I had another spare kidney, I'd do it again," Tracey tells the BBC's Victoria Derbyshire programme.
+She is what is known as an "altruistic donor"—someone willing to give away an organ to potentially help save the life of a complete stranger.
+A microbiologist in the NHS and the daughter of two nurses, she has spent her life learning about the importance of healthcare from a professional standpoint. But she has also been keen to make a difference on a personal level. "I signed up to donate blood and to the bone marrow register when I was 18," she says.
+Now 50, her wish to donate has become gradually more expansive. In 2012, she was one of fewer than 100 people that year to donate a kidney without knowing the recipient's identity, and she now supports the charity Give A Kidney, encouraging others to do the same.
+As of 30 September 2016, 5,126 people remain on the NHS kidney transplant waiting list. Tracey's kidney donation, in all likelihood, has saved someone's life. "I remind myself of it every day when I wake up," she says, rightly proud of her life-changing actions.
+It was not, however, a decision taken on the spur of a moment. Donating a kidney is an "involved process," she says, with suitability assessments taking at least three months to complete. Tests leading up to the transplant include X-rays, heart tracing, and a special test of kidney function, which involves an injection and a series of blood tests. "It is not something to do if you're scared of needles," she jokes.
+The risks associated with donating, however, are relatively low for those deemed healthy enough to proceed, with a mortality rate of about one in 3,000—roughly the same as having an appendix removed. Compared with the general public, NHS Blood and Transplant says, most kidney donors have equivalent—or better—life expectancy than the average person.
+Tracey says she was in hospital for five days after her operation but felt "back to normal" within six weeks.
+"""
+
+HUMAN_BBC_NEWS2 = """
+A message of hope at Washington march
+For such a divisive figure, Donald Trump managed to unify hundreds of thousands of Americans at the Women's March on Washington.
+Moments after Mr Trump was sworn in as the 45th president on Friday, he delivered a thundering speech in which he promised to improve the lives of millions of Americans.
+A day later, throngs of women, men and children streamed into the same area where he made that pledge, in order to take a stand for gender and racial equality.
+Though Mr Trump's named was mentioned frequently, the march, which organisers estimate attracted more than half a million, was not only about the new US president.
+Messages ranged from "Thank you for making me an activist Trump" to "We will not be silenced," but the common thread throughout the patchwork of signs was hope.
+"It's about solidarity and visualising the resistance," said Jonathon Meier, who took a bus from New York.
+"And I think it not only helps with the healing process, but it gives me hope for the next four years."
+A sea of activists, some clad in knitted, pink "pussy" hats and others draped in American flags, ambled about the National Mall, stopping to catch a glimpse of some of the high-profile speakers and singing along to songs like "This Little Light of Mine".
+Peppered among the many protest signs were images of ovaries and female genitals, a nod to concerns over losing access to birth control and abortion care under a Trump administration.
+"""
+
+FREELY_GENERATION_NEWS = """
+A new study has indicated that criminals and terrorists are increasingly turning to the dark net to purchase weapons. The study, conducted by cybersecurity firm Recorded Future, found that these purchases are being made anonymously and with cryptocurrency, making it difficult for law enforcement agencies to track and intercept them. The dark net is a hidden part of the internet, accessible only through anonymous browsers, where users can buy and sell a variety of illegal goods and services. However, the study found that weapons purchases are becoming more popular on the dark net, with firearms and explosives being the most commonly traded items. Recorded Future's research showed that many of the weapons being sold on the dark net are military-grade, and the study suggests that this is due to the large number of surplus weapons available following military conflicts in various parts of the world. The report also found that the sellers on the dark net are often located in countries with lax gun laws, leading to concerns that these weapons could end up in the hands of criminals and terrorists who could use them to commit acts of violence. The use of cryptocurrency to purchase these weapons adds another layer of difficulty for law enforcement agencies trying to track down those responsible. The anonymity provided by cryptocurrency allows buyers and sellers to conduct their transactions without leaving a trace. The findings of this study serve as a stark reminder of the dangers posed by the dark net, and the need for law enforcement agencies to remain vigilant in their efforts to combat illegal activity on this hidden part of the internet.
+"""
+
+HUMAN_BBC_NEWS2_IMAGE = "human_bbc_news_2.webp"
+
+HIGHLIGHT = "highlight"
+
+
+def highlight_text(words, indexes):
+    final_words = words
+    for index in indexes:
+        final_words[index] = (
+            f"<span style='color:#00FF00; font-weight:bold;'>{words[index]}</span>"
+        )
+    return " ".join(final_words)
+
+
+def format_pair(pair):
+    input_sentence = highlight_text(pair[0], pair[2])
+    source_sentence = highlight_text(pair[1], pair[3])
+    return f"<tr><td>{input_sentence}</td><td>{source_sentence}</td></tr>"
+
+
+def create_table(data):
+    table_rows = "\n".join([format_pair(pair) for pair in data])
+    return f"""
+    <h5> Comparison between input news and source news at the above link</h5>
+    <table border="1" style="width:100%; text-align:left; border-collapse:collapse;">
+        <thead>
+            <tr>
+                <th>Input sentence</th>
+                <th>Source sentence</th>
+            </tr>
+        </thead>
+        <tbody>
+            {table_rows}
+        </tbody>
+    </table>
+    """
+
+
+with gr.Blocks() as demo:
+    image = gr.Image(
+        value=HUMAN_IMAGE,
+        label="News Image",
+        height=200,
+        width=200,
+        type="filepath",
+    )
+    content = gr.Textbox(label="Content", lines=3, value=HUMAN_CONTENT)
+
+    process_btn = gr.Button("Process")
+
+    """
+        1. human bbc news
+        2. proofreading
+        3. opposite
+        4. human bbc news 2
+        5. human_cnn news
+        6. paraphrase
+        7. freely generation
+    """
+    gr.Examples(
+        examples=[
+            [HUMAN_IMAGE, HUMAN_CONTENT],
+            [MACHINE_IMAGE, MACHINE_CONTENT],
+            [MACHINE_IMAGE, OPPOSITE_NEWS],
+            [HUMAN_BBC_NEWS2_IMAGE, HUMAN_BBC_NEWS2],
+            [HUMAN_NEWS_CNN_IMAGE, HUMAN_NEWS_CNN],
+            [MACHINE_IMAGE, PARAPHASE_NEWS],
+            [MACHINE_IMAGE, FREELY_GENERATION_NEWS],
+        ],
+        inputs=[image, content],
+        label="examples",
+        example_labels=[
+            "human bbc news",
+            "proofreading",
+            "opposite",
+            "human bbc news 2",
+            "human cnn news",
+            "paraphrase",
+            "freely generation",
+        ],
+    )
+
+    overall = gr.HTML()
+    matching_html = gr.HTML()
+
+    def process(input_image, content):
+        (
+            search_engine_prediction,
+            SOTA_prediction,
+            SOTA_confidence,
+            found_url,
+            sentence_pairs,
+        ) = abstract_detect_generated_text(content)
+
+        final_table = []
+        COLOR_MAPS = {
+            "HUMAN": "<span style='color:#FFFF00'>",
+            "MACHINE": "<span style='color:red'>",
+        }
+
+        source_image = []
+        image_prediction_label, image_confidence = image_generation_detection(
+            input_image,
+        )
+        # [found_img_url, image_different_score] = find_similar_img_from_url(input_image)
+
+        # if 0 < image_different_score < 10:
+        #     search_engine_description = f'Most likely generated by {COLOR_MAPS["HUMAN"]} (score = {image_different_score})</span> with evidence link at <a href="{found_img_url}">{found_img_url} </a>'
+        # else:  # TODO add < 25 which is cropped images
+        #     search_engine_description = f'Most likely generated by {COLOR_MAPS["MACHINE"]} (score = {image_different_score})</span></a>'
+
+        for (
+            input_sentence,
+            source_sentence,
+            check_paraphrase,
+        ) in sentence_pairs:
+            input_words, source_words, input_indexes, source_indexes = (
+                highlight_overlap_by_word_to_list(
+                    input_sentence,
+                    source_sentence,
+                )
+            )
+            final_table.append(
+                (input_words, source_words, input_indexes, source_indexes),
+            )
+
+        if search_engine_prediction == UNKNOWN:
+            search_engine_description = "Cannot find any evidence link"
+            final_prediction = SOTA_prediction
+        else:
+            final_prediction = search_engine_prediction
+            search_engine_description = f'Most likely generated by {COLOR_MAPS[search_engine_prediction]}{search_engine_prediction}</span> with evidence link at <a href="{found_url}">{found_url} </a>'
+
+        overall_html_result = f"""
+<h1>Image generation detection</h1>
+<ul>
+    <li><strong>Prediction by SOTA method (provided by BDI members):</strong> Most likely generated by {COLOR_MAPS[image_prediction_label]}{image_prediction_label} </span>with confidence = {image_confidence}%</li>
+    <li><strong>Prediction by our method (developed by BDI members)</strong>: {search_engine_description}
+</ul>
+<hr>
+<h1>Text generation detection</h1>
+<ul>
+    <li><strong>Prediction by SOTA method (https://github.com/Hello-SimpleAI/chatgpt-comparison-detection)</strong>: Most likely generated by {COLOR_MAPS[SOTA_prediction]}{SOTA_prediction} </span>with confidence = {SOTA_confidence}</li>
+    <li><strong>Prediction by our method (developed by BDI members)</strong>: {search_engine_description}
+    <li><strong>Final prediction by our method (developed by BDI members)</strong>: Most likely generated by {COLOR_MAPS[final_prediction]}{final_prediction}&nbsp;</span></li>
+</ul>
+<p>&nbsp;</p>
+"""
+        if len(final_table) != 0:
+            html_table = create_table(final_table)
+        else:
+            html_table = ""
+        return overall_html_result, html_table
+
+    process_btn.click(
+        process,
+        inputs=[image, content],
+        outputs=[overall, matching_html],
+    )
+
+demo.launch(share=False)

pyproject.toml

@@ -0,0 +1,21 @@
+[tool.black]
+line-length = 79
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.idea
+  | \.pytest_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''
+
+[tool.isort]
+profile = "black"
+force_grid_wrap=2
+multi_line_output=3

readme.md

@@ -0,0 +1,70 @@
+# [Text] SimLLM: Detecting Sentences Generated by Large Language Models Using Similarity between the Generation and its Re-Generation
+
+## **Getting Started**
+1. **Clone the repository:**
+  ```bash
+  git clone https://github.com/Tokyo-Techies/prj-nict-ai-content-detection
+  ```
+
+2. **Set up the environment:**
+Using virtual environment:
+  ```bash
+  python -m venv .venv
+  source .venv/bin/activate
+  ```
+
+3. **Install dependencies:**
+  - Torch: https://pytorch.org/get-started/locally/
+  - Others
+  ```bash
+  pip install -r requirements.txt
+  ```
+
+
+1. **API Keys** (optional)
+   - Obtain API keys for the corresponding models and insert them into the `SimLLM.py` file:
+     - ChatGPT: [OpenAI API](https://openai.com/index/openai-api/)
+     - Gemini: [Google Gemini API](https://ai.google.dev/gemini-api/docs/api-key)
+     - Other LLMs: [Together API](https://api.together.ai/)
+
+
+5. **Run the project:**
+  - Text only:
+  ```bash
+  python SimLLM.py
+  ```
+
+### Parameters
+
+- `LLMs`: List of large language models to use. Available models include 'ChatGPT', 'Yi', 'OpenChat', 'Gemini', 'LLaMa', 'Phi', 'Mixtral', 'QWen', 'OLMO', 'WizardLM', and 'Vicuna'. Default is `['ChatGPT', 'Yi', 'OpenChat']`.
+- `train_indexes`: List of LLM indexes for training. Default is `[0, 1, 2]`.
+- `test_indexes`: List of LLM indexes for testing. Default is `[0]`.
+- `num_samples`: Number of samples. Default is 5000.
+
+### Examples
+
+- Running with default parameters:
+  `python SimLLM.py`
+
+- Running with customized parameters:
+  `python SimLLM.py --LLMs ChatGPT --train_indexes 0 --test_indexes 0`
+
+## Dataset
+
+The `dataset.csv` file contains both human and generated texts from 12 large language models, including:
+ChatGPT, GPT-4o, Yi, OpenChat, Gemini, LLaMa, Phi, Mixtral, QWen, OLMO, WizardLM, and Vicuna.
+
+## Citation
+
+```bibtex
+@inproceedings{nguyen2024SimLLM,
+  title={SimLLM: Detecting Sentences Generated by Large Language Models Using Similarity between the Generation and its Re-generation},
+  author={Nguyen-Son, Hoang-Quoc and Dao, Minh-Son and Zettsu, Koji},
+  booktitle={The Conference on Empirical Methods in Natural Language Processing},
+  year={2024}
+}
+```
+
+## Acknowledgements
+
+- BARTScore: [BARTScore GitHub Repository](https://github.com/neulab/BARTScore)

requirements.txt

@@ -0,0 +1,34 @@
+gradio
+# TEXT
+accelerate==1.1.1
+datasets==v2.11.0
+evaluate==0.4.3
+google-generativeai==0.8.3
+clean-text==0.6.0
+#grpc
+grpcio==1.68.1
+langchain==0.3.9
+langchain_community==0.3.8
+nltk==3.9.1
+openai==1.55.3
+rouge==1.0.1
+proto-plus==1.25.0
+scikit-learn==1.5.2
+sentence_transformers==3.3.1
+transformers==4.46.3
+xgboost==2.1.3
+
+# IMAGES
+torch==2.2.2
+torchvision==0.17.2
+pytorch-lightning==2.4.0
+timm==1.0.12
+opencv-python==4.10.0.84
+torchdata==0.7.1
+invisible-watermark==0.2.0
+google-image-source-search==1.2.2
+requests==2.31.0
+ImageHash==4.3.1
+
+# parsing
+PyPDF2

src/application/url_reader.py

@@ -0,0 +1,76 @@
+import string
+from bs4 import BeautifulSoup
+from newspaper import article, ArticleException, ArticleBinaryDataException
+import requests
+
+class URLReader():
+    def __init__(self, url: string, newspaper: bool=True):
+        self.url = url
+        self.text = None  # string
+        self.title = None  # string
+        self.images = None  # list of Image objects
+        self.top_image = None  # Image object
+        self.newspaper = newspaper  # True if using newspaper4k, False if using BS
+        if self.newspaper is True:
+            self.extract_content_newspaper()
+        else:
+            self.extract_content_bs()
+        
+    def extract_content_newspaper(self):
+        """
+        Use newspaper4k to extracts content from a URL
+
+        Args:
+            url: The URL of the web page.
+
+        Returns:
+            The extracted content (title, text, images)
+        """
+        
+        try:
+            response = requests.get(self.url)
+            response.raise_for_status()  # Raise exception for unsuccessful requests
+        except requests.exceptions.RequestException as e:
+            print(f"Error fetching URL: {e}")
+            return None
+        
+        try:
+            news = article(url=self.url, fetch_images=True)
+        except (ArticleException, ArticleBinaryDataException) as e:
+            print(f"\t\t↑↑↑ Error downloading article: {e}")
+            return None
+        
+        self.title = news.title
+        self.text = news.text
+        self.images = news.images
+        self.top_image = news.top_image
+
+    def extract_content_bs(self):
+        """
+        Use BS and process content
+        """
+        response = requests.get(self.url)
+        response.raise_for_status()
+        
+        response.encoding = response.apparent_encoding
+        
+        try:
+            soup = BeautifulSoup(response.content, "html.parser")
+        except:
+            print(f"Error parsing HTML content from {self.url}")
+            return None
+        
+        self.title = soup.title.string.strip() if soup.title else None
+        
+        image_urls = [img['src'] for img in soup.find_all('img')] 
+        self.images = image_urls
+        self.top_image = self.images[0]
+        
+        # Exclude text within specific elements
+        for element in soup(["img", "figcaption", "table", "script", "style"]):
+            element.extract()
+        #text = soup.get_text(separator="\n")
+        paragraphs = soup.find_all('p')
+        text = ' '.join([p.get_text() for p in paragraphs])
+
+        self.text = text

src/images/CNN_model_classifier.py

@@ -0,0 +1,63 @@
+import argparse
+
+import torch.nn
+import torchvision.transforms as transforms
+from PIL import Image
+
+from .CNN.networks.resnet import resnet50
+
+
+def predict_cnn(image, model_path, crop=None):
+    model = resnet50(num_classes=1)
+    state_dict = torch.load(model_path, map_location="cpu")
+    model.load_state_dict(state_dict["model"])
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model.to(device)
+    model.eval()
+
+    # Transform
+    if crop is not None:
+        trans_init = [transforms.CenterCrop(crop)]
+        print("Cropping to [%i]" % crop)
+        trans = transforms.Compose(
+            trans_init
+            + [
+                transforms.ToTensor(),
+                transforms.Normalize(
+                    mean=[0.485, 0.456, 0.406],
+                    std=[0.229, 0.224, 0.225],
+                ),
+            ],
+        )
+
+        image = trans(image.convert("RGB"))
+
+    with torch.no_grad():
+        in_tens = image.unsqueeze(0)
+        prob = model(in_tens).sigmoid().item()
+
+    return prob
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("-f", "--file", default="examples_realfakedir")
+    parser.add_argument(
+        "-m",
+        "--model_path",
+        type=str,
+        default="weights/blur_jpg_prob0.5.pth",
+    )
+    parser.add_argument(
+        "-c",
+        "--crop",
+        type=int,
+        default=None,
+        help="by default, do not crop. specify crop size",
+    )
+
+    opt = parser.parse_args()
+    prob = predict_cnn(Image.open(opt.file), opt.model_path, crop=opt.crop)
+    print(f"probability of being synthetic: {prob * 100:.2f}%")

src/images/Diffusion/Pipfile

@@ -0,0 +1,29 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[[source]]
+url = "https://download.pytorch.org/whl/cu121"
+verify_ssl = true
+name = "downloadpytorch"
+
+[packages]
+pandas = "*"
+numpy = "*"
+polars = "*"
+requests = "*"
+img2dataset = "*"
+torch = {version = "==2.1.0", index = "downloadpytorch"}
+torchvision = {version = "==0.16.0", index = "downloadpytorch"}
+lightning = "*"
+webdataset = "*"
+matplotlib = "*"
+invisible-watermark = "*"
+torchdata = "*"
+timm = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.11"

src/images/Diffusion/README.md

@@ -0,0 +1,72 @@
+# AI-generated image detection
+
+This is a group project developed by a team of two individuals.
+
+## Managing Python packages
+
+Use of `pipenv` is recommended. The required packages are in `Pipfile`, and can be installed using `pipenv install`.
+
+## Scraping script for Reddit
+
+`python scrape.py --subreddit midjourney --flair Showcase`
+
+This command will scrape the midjourney subreddit, and filter posts that contain the "Showcase" flair. The default number of images to scrape is 30000. The output will contain a parquet file containing metadata, and a csv file containing the urls.
+
+`img2dataset --url_list=urls/midjourney.csv --output_folder=data/midjourney --thread_count=64 --resize_mode=no --output_format=webdataset`
+
+This command will download the images in the webdataset format.
+
+
+## Laion script for real images
+
+`wget -l1 -r --no-parent https://the-eye.eu/public/AI/cah/laion400m-met-release/laion400m-meta/
+mv the-eye.eu/public/AI/cah/laion400m-met-release/laion400m-meta/ .`
+
+This command will download a 50GB url metadata dataset in 32 parquet files.
+
+`sample_laion_script.ipynb`
+
+This script consolidates the parquet files, excludes NSFW images, and selects a subset of 224,917 images.
+
+`combine_laion_script`
+
+This script combines the outputs from earlier into 1 parquet file.
+
+`img2dataset --url_list urls/laion.parquet --input_format "parquet" --url_col "URL" --caption_col "TEXT" --skip_reencode True --output_format webdataset --output_folder data/laion400m_data --processes_count 16 --thread_count 128 --resize_mode no --save_additional_columns '["NSFW","similarity","LICENSE"]' --enable_wandb True`
+
+This command will download the images in the webdataset format.
+
+
+## Data splitting, preprocessing and loading
+
+`data_split.py` splits the data according to 80/10/10. The number of samples:
+
+```
+./data/laion400m_data: (115346, 14418, 14419)
+./data/genai-images/StableDiffusion: (22060, 2757, 2758)
+./data/genai-images/midjourney: (21096, 2637, 2637)
+./data/genai-images/dalle2: (13582, 1697, 1699)
+./data/genai-images/dalle3: (12027, 1503, 1504)
+```
+
+Each sample contains image, target label(1 for GenAI images), and domain label(denoting which generator the image is from). The meaning of the domain label is:
+
+```
+DOMAIN_LABELS = {
+    0: "laion",
+    1: "StableDiffusion",
+    2: "dalle2",
+    3: "dalle3",
+    4: "midjourney"
+}
+```
+
+The `load_dataloader()` function in `dataloader.py` returns a `torchdata.dataloader2.DataLoader2` given a list of domains for GenAI images(subset of `[1, 2, 3, 4]`, LAION will always be included). When building the training dataset, data augmentation and class balanced sampling are applied. It is very memory intensive(>20G) and takes some time to fill its buffer before producing batches. Use the dataloader in this way:
+
+```
+for epoch in range(10):
+    dl.seed(epoch)
+    for d in dl:
+        model(d)
+dl.shutdown()
+```

src/images/Diffusion/combine_laion_script.ipynb

@@ -0,0 +1,117 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pip install pyspark"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "current_directory = os.getcwd()\n",
+    "print(current_directory)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.chdir(current_directory)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from pyspark.sql import SparkSession\n",
+    "from pyspark.sql.functions import col\n",
+    "\n",
+    "spark = SparkSession.builder.appName(\"CombineParquetFiles\").config(\"spark.executor.memory\", \"8g\").config(\"spark.executor.cores\", \"4\").config(\"spark.executor.instances\", \"3\").config(\"spark.dynamicAllocation.enabled\", \"true\").config(\"spark.task.maxFailures\", 10).getOrCreate()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "parquet_directory_path = '/Users/fionachow/Documents/NYU/CDS/Fall 2023/CSCI - GA 2271 - Computer Vision/Project/laion_sampled'\n",
+    "\n",
+    "output_parquet_file = '/Users/fionachow/Documents/NYU/CDS/Fall 2023/CSCI - GA 2271 - Computer Vision/Project/laion_combined'\n",
+    "\n",
+    "df = spark.read.parquet(parquet_directory_path)\n",
+    "\n",
+    "df_coalesced = df.coalesce(1)\n",
+    "\n",
+    "df_coalesced.write.mode('overwrite').parquet(output_parquet_file)\n",
+    "\n",
+    "row_count = df.count()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(row_count)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "parquet_directory_path = '/Users/fionachow/Documents/NYU/CDS/Fall 2023/CSCI - GA 2271 - Computer Vision/Project/laion_combined/part-00000-0190eea7-02ac-4ea0-86fd-0722308c0c58-c000.snappy.parquet'\n",
+    "\n",
+    "df = spark.read.parquet(parquet_directory_path)\n",
+    "\n",
+    "df.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(df.count())"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "bloom",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/images/Diffusion/data_split.py

@@ -0,0 +1,80 @@
+import glob
+import json
+
+import webdataset as wds
+
+
+def split_dataset(path, n_train, n_val, n_test, label, domain_label):
+    max_file_size = 1000
+    input_files = glob.glob(path + "/*.tar")
+    src = wds.WebDataset(input_files)
+
+    train_path_prefix = path + "/train"
+    val_path_prefix = path + "/val"
+    test_path_prefix = path + "/test"
+
+    def write_split(dataset, prefix, start, end):
+        n_split = end - start
+        output_files = [
+            f"{prefix}_{i}.tar" for i in range(n_split // max_file_size + 1)
+        ]
+        for i, output_file in enumerate(output_files):
+            print(f"Writing {output_file}")
+            with wds.TarWriter(output_file) as dst:
+                for sample in dataset.slice(
+                    start + i * max_file_size,
+                    min(start + (i + 1) * max_file_size, end),
+                ):
+                    new_sample = {
+                        "__key__": sample["__key__"],
+                        "jpg": sample["jpg"],
+                        "label.cls": label,
+                        "domain_label.cls": domain_label,
+                    }
+                    dst.write(new_sample)
+
+    write_split(src, train_path_prefix, 0, n_train)
+    write_split(src, val_path_prefix, n_train, n_train + n_val)
+    write_split(
+        src,
+        test_path_prefix,
+        n_train + n_val,
+        n_train + n_val + n_test,
+    )
+
+
+def calculate_sizes(path):
+    stat_files = glob.glob(path + "/*_stats.json")
+    total = 0
+    for f in stat_files:
+        with open(f) as stats:
+            total += json.load(stats)["successes"]
+    n_train = int(total * 0.8)
+    n_val = int(total * 0.1)
+    n_test = total - n_train - n_val
+
+    return n_train, n_val, n_test
+
+
+if __name__ == "__main__":
+
+    paths = [
+        "./data/laion400m_data",
+        "./data/genai-images/StableDiffusion",
+        "./data/genai-images/midjourney",
+        "./data/genai-images/dalle2",
+        "./data/genai-images/dalle3",
+    ]
+
+    sizes = []
+    for p in paths:
+        res = calculate_sizes(p)
+        sizes.append(res)
+
+    domain_labels = [0, 1, 4, 2, 3]
+
+    for i, p in enumerate(paths):
+        print(f"{p}: {sizes[i]}")
+        label = 0 if i == 0 else 1
+        print(label, domain_labels[i])
+        split_dataset(p, *calculate_sizes(p), label, domain_labels[i])

src/images/Diffusion/dataloader.py

@@ -0,0 +1,228 @@
+import argparse
+import collections
+import random
+from typing import Iterator
+
+import cv2
+import numpy as np
+import torchdata.datapipes as dp
+from imwatermark import WatermarkEncoder
+from PIL import (
+    Image,
+    ImageFile,
+)
+from torch.utils.data import DataLoader
+from torchdata.datapipes.iter import (
+    Concater,
+    FileLister,
+    FileOpener,
+    SampleMultiplexer,
+)
+from torchvision.transforms import v2
+from tqdm import tqdm
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+Image.MAX_IMAGE_PIXELS = 1000000000
+
+encoder = WatermarkEncoder()
+encoder.set_watermark("bytes", b"test")
+
+
+DOMAIN_LABELS = {
+    0: "laion",
+    1: "StableDiffusion",
+    2: "dalle2",
+    3: "dalle3",
+    4: "midjourney",
+}
+
+N_SAMPLES = {
+    0: (115346, 14418, 14419),
+    1: (22060, 2757, 2758),
+    4: (21096, 2637, 2637),
+    2: (13582, 1697, 1699),
+    3: (12027, 1503, 1504),
+}
+
+
+@dp.functional_datapipe("collect_from_workers")
+class WorkerResultCollector(dp.iter.IterDataPipe):
+    def __init__(self, source: dp.iter.IterDataPipe):
+        self.source = source
+
+    def __iter__(self) -> Iterator:
+        yield from self.source
+
+    def is_replicable(self) -> bool:
+        """Method to force data back to main process"""
+        return False
+
+
+def crop_bottom(image, cutoff=16):
+    return image[:, :-cutoff, :]
+
+
+def random_gaussian_blur(image, p=0.01):
+    if random.random() < p:
+        return v2.functional.gaussian_blur(image, kernel_size=5)
+    return image
+
+
+def random_invisible_watermark(image, p=0.2):
+    image_np = np.array(image)
+    image_np = np.transpose(image_np, (1, 2, 0))
+
+    if image_np.ndim == 2:  # Grayscale image
+        image_np = cv2.cvtColor(image_np, cv2.COLOR_GRAY2BGR)
+    elif image_np.shape[2] == 4:  # RGBA image
+        image_np = cv2.cvtColor(image_np, cv2.COLOR_RGBA2BGR)
+
+    # print(image_np.shape)
+    if image_np.shape[0] < 256 or image_np.shape[1] < 256:
+        image_np = cv2.resize(
+            image_np,
+            (256, 256),
+            interpolation=cv2.INTER_AREA,
+        )
+    if random.random() < p:
+        return encoder.encode(image_np, method="dwtDct")
+    return image_np
+
+
+def build_transform(split: str):
+    train_transform = v2.Compose(
+        [
+            v2.Lambda(crop_bottom),
+            v2.RandomCrop((256, 256), pad_if_needed=True),
+            v2.Lambda(random_gaussian_blur),
+            v2.RandomGrayscale(p=0.05),
+            v2.Lambda(random_invisible_watermark),
+            v2.ToImage(),
+        ],
+    )
+
+    eval_transform = v2.Compose(
+        [
+            v2.CenterCrop((256, 256)),
+        ],
+    )
+    transform = train_transform if split == "train" else eval_transform
+
+    return transform
+
+
+def dp_to_tuple_train(input_dict):
+    transform = build_transform("train")
+    return (
+        transform(input_dict[".jpg"]),
+        input_dict[".label.cls"],
+        input_dict[".domain_label.cls"],
+    )
+
+
+def dp_to_tuple_eval(input_dict):
+    transform = build_transform("eval")
+    return (
+        transform(input_dict[".jpg"]),
+        input_dict[".label.cls"],
+        input_dict[".domain_label.cls"],
+    )
+
+
+def load_dataset(domains: list[int], split: str):
+
+    laion_lister = FileLister("./data/laion400m_data", f"{split}*.tar")
+    genai_lister = {
+        d: FileLister(
+            f"./data/genai-images/{DOMAIN_LABELS[d]}",
+            f"{split}*.tar",
+        )
+        for d in domains
+        if DOMAIN_LABELS[d] != "laion"
+    }
+    weight_genai = 1 / len(genai_lister)
+
+    def open_lister(lister):
+        opener = FileOpener(lister, mode="b")
+        return opener.load_from_tar().routed_decode().webdataset()
+
+    buffer_size1 = 100 if split == "train" else 10
+    buffer_size2 = 100 if split == "train" else 10
+
+    if split != "train":
+        all_lister = [laion_lister] + list(genai_lister.values())
+        dp = open_lister(Concater(*all_lister)).sharding_filter()
+    else:
+        laion_dp = (
+            open_lister(laion_lister.shuffle())
+            .cycle()
+            .sharding_filter()
+            .shuffle(buffer_size=buffer_size1)
+        )
+        genai_dp = {
+            open_lister(genai_lister[d].shuffle())
+            .cycle()
+            .sharding_filter()
+            .shuffle(buffer_size=buffer_size1): weight_genai
+            for d in domains
+            if DOMAIN_LABELS[d] != "laion"
+        }
+        dp = SampleMultiplexer({laion_dp: 1, **genai_dp}).shuffle(
+            buffer_size=buffer_size2,
+        )
+
+    if split == "train":
+        dp = dp.map(dp_to_tuple_train)
+    else:
+        dp = dp.map(dp_to_tuple_eval)
+
+    return dp
+
+
+def load_dataloader(
+    domains: list[int],
+    split: str,
+    batch_size: int = 32,
+    num_workers: int = 4,
+):
+    dp = load_dataset(domains, split)
+    # if split == "train":
+    #     dp = UnderSamplerIterDataPipe(dp, {0: 0.5, 1: 0.5}, seed=42)
+    dp = dp.batch(batch_size).collate()
+    dl = DataLoader(
+        dp,
+        batch_size=None,
+        num_workers=num_workers,
+        pin_memory=True,
+    )
+
+    return dl
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    args = parser.parse_args()
+
+    # testing code
+    dl = load_dataloader([0, 1], "train", num_workers=8)
+    y_dist = collections.Counter()
+    d_dist = collections.Counter()
+
+    for i, (img, y, d) in tqdm(enumerate(dl)):
+        if i % 100 == 0:
+            print(y, d)
+        if i == 400:
+            break
+        y_dist.update(y.numpy())
+        d_dist.update(d.numpy())
+
+    print("class label")
+    for label in sorted(y_dist):
+        frequency = y_dist[label] / sum(y_dist.values())
+        print(f"• {label}: {frequency:.2%} ({y_dist[label]})")
+
+    print("domain label")
+    for label in sorted(d_dist):
+        frequency = d_dist[label] / sum(d_dist.values())
+        print(f"• {label}: {frequency:.2%} ({d_dist[label]})")

src/images/Diffusion/diffusion_data_loader.py

@@ -0,0 +1,233 @@
+import argparse
+import collections
+import glob
+import os
+import random
+from typing import Iterator
+
+import cv2
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchdata as td
+import torchdata.datapipes as dp
+from imwatermark import WatermarkEncoder
+from PIL import (
+    Image,
+    ImageFile,
+)
+from torch.utils.data import (
+    DataLoader,
+    RandomSampler,
+)
+from torchdata.dataloader2 import (
+    DataLoader2,
+    MultiProcessingReadingService,
+)
+from torchdata.datapipes.iter import (
+    Concater,
+    FileLister,
+    FileOpener,
+    SampleMultiplexer,
+)
+from torchvision.transforms import v2
+from tqdm import tqdm
+from utils_sampling import UnderSamplerIterDataPipe
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+Image.MAX_IMAGE_PIXELS = 1000000000
+
+encoder = WatermarkEncoder()
+encoder.set_watermark("bytes", b"test")
+
+DOMAIN_LABELS = {
+    0: "laion",
+    1: "StableDiffusion",
+    2: "dalle2",
+    3: "dalle3",
+    4: "midjourney",
+}
+
+N_SAMPLES = {
+    0: (115346, 14418, 14419),
+    1: (22060, 2757, 2758),
+    4: (21096, 2637, 2637),
+    2: (13582, 1697, 1699),
+    3: (12027, 1503, 1504),
+}
+
+
+@dp.functional_datapipe("collect_from_workers")
+class WorkerResultCollector(dp.iter.IterDataPipe):
+    def __init__(self, source: dp.iter.IterDataPipe):
+        self.source = source
+
+    def __iter__(self) -> Iterator:
+        yield from self.source
+
+    def is_replicable(self) -> bool:
+        """Method to force data back to main process"""
+        return False
+
+
+def crop_bottom(image, cutoff=16):
+    return image[:, :-cutoff, :]
+
+
+def random_gaussian_blur(image, p=0.01):
+    if random.random() < p:
+        return v2.functional.gaussian_blur(image, kernel_size=5)
+    return image
+
+
+def random_invisible_watermark(image, p=0.2):
+    image_np = np.array(image)
+    image_np = np.transpose(image_np, (1, 2, 0))
+
+    if image_np.ndim == 2:  # Grayscale image
+        image_np = cv2.cvtColor(image_np, cv2.COLOR_GRAY2BGR)
+    elif image_np.shape[2] == 4:  # RGBA image
+        image_np = cv2.cvtColor(image_np, cv2.COLOR_RGBA2BGR)
+
+    # print(image_np.shape)
+    if image_np.shape[0] < 256 or image_np.shape[1] < 256:
+        image_np = cv2.resize(
+            image_np, (256, 256), interpolation=cv2.INTER_AREA
+        )
+    if random.random() < p:
+        return encoder.encode(image_np, method="dwtDct")
+    return image_np
+
+
+def build_transform(split: str):
+    train_transform = v2.Compose(
+        [
+            v2.Lambda(crop_bottom),
+            v2.RandomCrop((256, 256), pad_if_needed=True),
+            v2.Lambda(random_gaussian_blur),
+            v2.RandomGrayscale(p=0.05),
+            v2.Lambda(random_invisible_watermark),
+            v2.ToImage(),
+        ]
+    )
+
+    eval_transform = v2.Compose(
+        [
+            v2.CenterCrop((256, 256)),
+        ]
+    )
+    transform = train_transform if split == "train" else eval_transform
+
+    return transform
+
+
+def dp_to_tuple_train(input_dict):
+    transform = build_transform("train")
+    return (
+        transform(input_dict[".jpg"]),
+        input_dict[".label.cls"],
+        input_dict[".domain_label.cls"],
+    )
+
+
+def dp_to_tuple_eval(input_dict):
+    transform = build_transform("eval")
+    return (
+        transform(input_dict[".jpg"]),
+        input_dict[".label.cls"],
+        input_dict[".domain_label.cls"],
+    )
+
+
+def load_dataset(domains: list[int], split: str):
+    laion_lister = FileLister("./data/laion400m_data", f"{split}*.tar")
+    genai_lister = {
+        d: FileLister(
+            f"./data/genai-images/{DOMAIN_LABELS[d]}", f"{split}*.tar"
+        )
+        for d in domains
+        if DOMAIN_LABELS[d] != "laion"
+    }
+    weight_genai = 1 / len(genai_lister)
+
+    def open_lister(lister):
+        opener = FileOpener(lister, mode="b")
+        return opener.load_from_tar().routed_decode().webdataset()
+
+    buffer_size1 = 100 if split == "train" else 10
+    buffer_size2 = 100 if split == "train" else 10
+
+    if split != "train":
+        all_lister = [laion_lister] + list(genai_lister.values())
+        dp = open_lister(Concater(*all_lister)).sharding_filter()
+    else:
+        laion_dp = (
+            open_lister(laion_lister.shuffle())
+            .cycle()
+            .sharding_filter()
+            .shuffle(buffer_size=buffer_size1)
+        )
+        genai_dp = {
+            open_lister(genai_lister[d].shuffle())
+            .cycle()
+            .sharding_filter()
+            .shuffle(
+                buffer_size=buffer_size1,
+            ): weight_genai
+            for d in domains
+            if DOMAIN_LABELS[d] != "laion"
+        }
+        dp = SampleMultiplexer({laion_dp: 1, **genai_dp}).shuffle(
+            buffer_size=buffer_size2
+        )
+
+    if split == "train":
+        dp = dp.map(dp_to_tuple_train)
+    else:
+        dp = dp.map(dp_to_tuple_eval)
+
+    return dp
+
+
+def load_dataloader(
+    domains: list[int], split: str, batch_size: int = 32, num_workers: int = 4
+):
+    dp = load_dataset(domains, split)
+    # if split == "train":
+    #     dp = UnderSamplerIterDataPipe(dp, {0: 0.5, 1: 0.5}, seed=42)
+    dp = dp.batch(batch_size).collate()
+    dl = DataLoader(
+        dp, batch_size=None, num_workers=num_workers, pin_memory=True
+    )
+
+    return dl
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    args = parser.parse_args()
+
+    # testing code
+    dl = load_dataloader([0, 1], "train", num_workers=8)
+    y_dist = collections.Counter()
+    d_dist = collections.Counter()
+
+    for i, (img, y, d) in tqdm(enumerate(dl)):
+        if i % 100 == 0:
+            print(y, d)
+        if i == 400:
+            break
+        y_dist.update(y.numpy())
+        d_dist.update(d.numpy())
+
+    print("class label")
+    for label in sorted(y_dist):
+        frequency = y_dist[label] / sum(y_dist.values())
+        print(f"• {label}: {frequency:.2%} ({y_dist[label]})")
+
+    print("domain label")
+    for label in sorted(d_dist):
+        frequency = d_dist[label] / sum(d_dist.values())
+        print(f"• {label}: {frequency:.2%} ({d_dist[label]})")

src/images/Diffusion/diffusion_model_classifier.py

@@ -0,0 +1,242 @@
+import argparse
+import logging
+import os
+
+import pandas as pd
+import pytorch_lightning as pl
+import timm
+import torch
+import torchvision.transforms as transforms
+from data_split import *
+from dataloader import *
+from PIL import Image
+from pytorch_lightning.callbacks import (
+    EarlyStopping,
+    ModelCheckpoint,
+)
+from sklearn.metrics import roc_auc_score
+from torchmetrics import (
+    Accuracy,
+    Recall,
+)
+from utils_sampling import *
+
+logging.basicConfig(
+    filename="training.log", filemode="w", level=logging.INFO, force=True
+)
+
+
+class ImageClassifier(pl.LightningModule):
+    def __init__(self, lmd=0):
+        super().__init__()
+        self.model = timm.create_model(
+            "resnet50", pretrained=True, num_classes=1
+        )
+        self.accuracy = Accuracy(task="binary", threshold=0.5)
+        self.recall = Recall(task="binary", threshold=0.5)
+        self.validation_outputs = []
+        self.lmd = lmd
+
+    def forward(self, x):
+        return self.model(x)
+
+    def training_step(self, batch):
+        images, labels, _ = batch
+        outputs = self.forward(images).squeeze()
+
+        print(f"Shape of outputs (training): {outputs.shape}")
+        print(f"Shape of labels (training): {labels.shape}")
+
+        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
+        logging.info(f"Training Step - ERM loss: {loss.item()}")
+        loss += self.lmd * (outputs**2).mean()  # SD loss penalty
+        logging.info(f"Training Step - SD loss: {loss.item()}")
+        return loss
+
+    def validation_step(self, batch):
+        images, labels, _ = batch
+        outputs = self.forward(images).squeeze()
+
+        if outputs.shape == torch.Size([]):
+            return
+
+        print(f"Shape of outputs (validation): {outputs.shape}")
+        print(f"Shape of labels (validation): {labels.shape}")
+
+        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
+        preds = torch.sigmoid(outputs)
+        self.log("val_loss", loss, prog_bar=True, sync_dist=True)
+        self.log(
+            "val_acc",
+            self.accuracy(preds, labels.int()),
+            prog_bar=True,
+            sync_dist=True,
+        )
+        self.log(
+            "val_recall",
+            self.recall(preds, labels.int()),
+            prog_bar=True,
+            sync_dist=True,
+        )
+        output = {"val_loss": loss, "preds": preds, "labels": labels}
+        self.validation_outputs.append(output)
+        logging.info(f"Validation Step - Batch loss: {loss.item()}")
+        return output
+
+    def predict_step(self, batch):
+        images, label, domain = batch
+        outputs = self.forward(images).squeeze()
+        preds = torch.sigmoid(outputs)
+        return preds, label, domain
+
+    def on_validation_epoch_end(self):
+        if not self.validation_outputs:
+            logging.warning("No outputs in validation step to process")
+            return
+        preds = torch.cat([x["preds"] for x in self.validation_outputs])
+        labels = torch.cat([x["labels"] for x in self.validation_outputs])
+        if labels.unique().size(0) == 1:
+            logging.warning("Only one class in validation step")
+            return
+        auc_score = roc_auc_score(labels.cpu(), preds.cpu())
+        self.log("val_auc", auc_score, prog_bar=True, sync_dist=True)
+        logging.info(f"Validation Epoch End - AUC score: {auc_score}")
+        self.validation_outputs = []
+
+    def configure_optimizers(self):
+        optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0005)
+        return optimizer
+
+
+checkpoint_callback = ModelCheckpoint(
+    monitor="val_loss",
+    dirpath="./model_checkpoints/",
+    filename="image-classifier-{step}-{val_loss:.2f}",
+    save_top_k=3,
+    mode="min",
+    every_n_train_steps=1001,
+    enable_version_counter=True,
+)
+
+early_stop_callback = EarlyStopping(
+    monitor="val_loss",
+    patience=4,
+    mode="min",
+)
+
+
+def load_image(image_path, transform=None):
+    image = Image.open(image_path).convert("RGB")
+
+    if transform:
+        image = transform(image)
+
+    return image
+
+
+def predict_single_image(image_path, model, transform=None):
+    image = load_image(image_path, transform)
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    model.to(device)
+
+    image = image.to(device)
+
+    model.eval()
+
+    with torch.no_grad():
+        image = image.unsqueeze(0)
+        output = model(image).squeeze()
+        print(output)
+        prediction = torch.sigmoid(output).item()
+
+    return prediction
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--ckpt_path", help="checkpoint to continue from", required=False
+)
+parser.add_argument(
+    "--predict", help="predict on test set", action="store_true"
+)
+parser.add_argument("--reset", help="reset training", action="store_true")
+parser.add_argument(
+    "--predict_image",
+    help="predict the class of a single image",
+    action="store_true",
+)
+parser.add_argument(
+    "--image_path",
+    help="path to the image to predict",
+    type=str,
+    required=False,
+)
+args = parser.parse_args()
+
+train_domains = [0, 1, 4]
+val_domains = [0, 1, 4]
+lmd_value = 0
+
+if args.predict:
+    test_dl = load_dataloader(
+        [0, 1, 2, 3, 4], "test", batch_size=128, num_workers=1
+    )
+    model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
+    trainer = pl.Trainer()
+    predictions = trainer.predict(model, dataloaders=test_dl)
+    preds, labels, domains = zip(*predictions)
+    preds = torch.cat(preds).cpu().numpy()
+    labels = torch.cat(labels).cpu().numpy()
+    domains = torch.cat(domains).cpu().numpy()
+    print(preds.shape, labels.shape, domains.shape)
+    df = pd.DataFrame({"preds": preds, "labels": labels, "domains": domains})
+    filename = "preds-" + args.ckpt_path.split("/")[-1]
+    df.to_csv(f"outputs/{filename}.csv", index=False)
+elif args.predict_image:
+    image_path = args.image_path
+    model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
+
+    # Define the transformations for the image
+    transform = transforms.Compose(
+        [
+            transforms.Resize((224, 224)),  # Image size expected by ResNet50
+            transforms.ToTensor(),
+            transforms.Normalize(
+                mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
+            ),
+        ]
+    )
+
+    prediction = predict_single_image(image_path, model, transform)
+    print("prediction", prediction)
+
+    # Output the prediction
+    print(
+        f"Prediction for {image_path}: {'Human' if prediction <= 0.001 else 'Generated'}"
+    )
+else:
+    train_dl = load_dataloader(
+        train_domains, "train", batch_size=128, num_workers=4
+    )
+    logging.info("Training dataloader loaded")
+    val_dl = load_dataloader(val_domains, "val", batch_size=128, num_workers=4)
+    logging.info("Validation dataloader loaded")
+
+    if args.reset:
+        model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
+    else:
+        model = ImageClassifier(lmd=lmd_value)
+    trainer = pl.Trainer(
+        callbacks=[checkpoint_callback, early_stop_callback],
+        max_steps=20000,
+        val_check_interval=1000,
+        check_val_every_n_epoch=None,
+    )
+    trainer.fit(
+        model=model,
+        train_dataloaders=train_dl,
+        val_dataloaders=val_dl,
+        ckpt_path=args.ckpt_path if not args.reset else None,
+    )

src/images/Diffusion/evaluation.ipynb

@@ -0,0 +1,187 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import polars as pl\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score, f1_score, RocCurveDisplay\n",
+    "\n",
+    "sns.set()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def pfbeta(labels, predictions, beta=1):\n",
+    "    y_true_count = 0\n",
+    "    ctp = 0\n",
+    "    cfp = 0\n",
+    "\n",
+    "    for idx in range(len(labels)):\n",
+    "        prediction = min(max(predictions[idx], 0), 1)\n",
+    "        if (labels[idx]):\n",
+    "            y_true_count += 1\n",
+    "            ctp += prediction\n",
+    "        else:\n",
+    "            cfp += prediction\n",
+    "\n",
+    "    beta_squared = beta * beta\n",
+    "    c_precision = ctp / (ctp + cfp)\n",
+    "    c_recall = ctp / y_true_count\n",
+    "    if (c_precision > 0 and c_recall > 0):\n",
+    "        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n",
+    "        return result\n",
+    "    else:\n",
+    "        return 0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_part_metrics(df: pl.DataFrame, threshold=0.3) -> dict:\n",
+    "    df = df.with_columns((df[\"preds\"] > threshold).alias(\"preds_bin\"))\n",
+    "    metrics = {}\n",
+    "    # binary metrics using the threshold\n",
+    "    metrics[\"accuracy\"] = accuracy_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
+    "    metrics[\"precision\"] = precision_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
+    "    metrics[\"recall\"] = recall_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
+    "    metrics[\"f1\"] = f1_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
+    "    # probabilistic F1 (doesn't depend on the threshold)\n",
+    "    metrics[\"pf1\"] = pfbeta(df[\"labels\"].to_numpy(), df[\"preds\"].to_numpy())\n",
+    "    # ROC AUC\n",
+    "    metrics[\"roc_auc\"] = roc_auc_score(df[\"labels\"].to_numpy(), df[\"preds\"].to_numpy())\n",
+    "    return metrics\n",
+    "\n",
+    "\n",
+    "def get_all_metrics(df: pl.DataFrame, threshold=0.3) -> pd.DataFrame:\n",
+    "    groups = [list(range(5)), [0, 1], [0, 4], [0, 2], [0, 3]]\n",
+    "    group_names = [\"all\", \"StableDiffusion\", \"Midjourney\", \"Dalle2\", \"Dalle3\"]\n",
+    "    all_metrics = []\n",
+    "    for i, g in enumerate(groups):\n",
+    "        subset = df.filter(pl.col(\"domains\").is_in(g))\n",
+    "        metrics = get_part_metrics(subset, threshold=threshold)\n",
+    "        metrics[\"group\"] = group_names[i]\n",
+    "        all_metrics.append(metrics)\n",
+    "    \n",
+    "    return pd.DataFrame(all_metrics)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df1 = pl.read_csv(\"outputs/preds-image-classifier-1.csv\")\n",
+    "metrics_df1 = get_all_metrics(df1, threshold=0.5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "metrics_df1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df14 = pl.read_csv(\"outputs/preds-image-classifier-14.csv\")\n",
+    "metrics_df14 = get_all_metrics(df14, threshold=0.5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "metrics_df14"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df142 = pl.read_csv(\"outputs/preds-image-classifier-142.csv\")\n",
+    "metrics_df142 = get_all_metrics(df142, threshold=0.5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "metrics_df142"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df1423 = pl.read_csv(\"outputs/preds-image-classifier-1423.csv\")\n",
+    "metrics_df1423 = get_all_metrics(df1423, threshold=0.5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "metrics_df1423"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "GenAI-image-detection-Z_9oJJe7",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/images/Diffusion/model.py

@@ -0,0 +1,307 @@
+import argparse
+import logging
+import os
+
+import pandas as pd
+import pytorch_lightning as pl
+import timm
+import torch
+import torch.nn.functional as F
+import torchvision.transforms as transforms
+from dataloader import load_dataloader
+from PIL import Image
+from pytorch_lightning.callbacks import (
+    EarlyStopping,
+    ModelCheckpoint,
+)
+from sklearn.metrics import roc_auc_score
+from torchmetrics import (
+    Accuracy,
+    Recall,
+)
+
+logging.basicConfig(
+    filename="training.log",
+    filemode="w",
+    level=logging.INFO,
+    force=True,
+)
+
+
+class ImageClassifier(pl.LightningModule):
+    def __init__(self, lmd=0):
+        super().__init__()
+        self.model = timm.create_model(
+            "resnet50",
+            pretrained=True,
+            num_classes=1,
+        )
+        self.accuracy = Accuracy(task="binary", threshold=0.5)
+        self.recall = Recall(task="binary", threshold=0.5)
+        self.validation_outputs = []
+        self.lmd = lmd
+
+    def forward(self, x):
+        return self.model(x)
+
+    def training_step(self, batch):
+        images, labels, _ = batch
+        outputs = self.forward(images).squeeze()
+
+        print(f"Shape of outputs (training): {outputs.shape}")
+        print(f"Shape of labels (training): {labels.shape}")
+
+        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
+        logging.info(f"Training Step - ERM loss: {loss.item()}")
+        loss += self.lmd * (outputs**2).mean()  # SD loss penalty
+        logging.info(f"Training Step - SD loss: {loss.item()}")
+        return loss
+
+    def validation_step(self, batch):
+        images, labels, _ = batch
+        outputs = self.forward(images).squeeze()
+
+        if outputs.shape == torch.Size([]):
+            return
+
+        print(f"Shape of outputs (validation): {outputs.shape}")
+        print(f"Shape of labels (validation): {labels.shape}")
+
+        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
+        preds = torch.sigmoid(outputs)
+        self.log("val_loss", loss, prog_bar=True, sync_dist=True)
+        self.log(
+            "val_acc",
+            self.accuracy(preds, labels.int()),
+            prog_bar=True,
+            sync_dist=True,
+        )
+        self.log(
+            "val_recall",
+            self.recall(preds, labels.int()),
+            prog_bar=True,
+            sync_dist=True,
+        )
+        output = {"val_loss": loss, "preds": preds, "labels": labels}
+        self.validation_outputs.append(output)
+        logging.info(f"Validation Step - Batch loss: {loss.item()}")
+        return output
+
+    def predict_step(self, batch):
+        images, label, domain = batch
+        outputs = self.forward(images).squeeze()
+        preds = torch.sigmoid(outputs)
+        return preds, label, domain
+
+    def on_validation_epoch_end(self):
+        if not self.validation_outputs:
+            logging.warning("No outputs in validation step to process")
+            return
+        preds = torch.cat([x["preds"] for x in self.validation_outputs])
+        labels = torch.cat([x["labels"] for x in self.validation_outputs])
+        if labels.unique().size(0) == 1:
+            logging.warning("Only one class in validation step")
+            return
+        auc_score = roc_auc_score(labels.cpu(), preds.cpu())
+        self.log("val_auc", auc_score, prog_bar=True, sync_dist=True)
+        logging.info(f"Validation Epoch End - AUC score: {auc_score}")
+        self.validation_outputs = []
+
+    def configure_optimizers(self):
+        optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0005)
+        return optimizer
+
+
+checkpoint_callback = ModelCheckpoint(
+    monitor="val_loss",
+    dirpath="./model_checkpoints/",
+    filename="image-classifier-{step}-{val_loss:.2f}",
+    save_top_k=3,
+    mode="min",
+    every_n_train_steps=1001,
+    enable_version_counter=True,
+)
+
+early_stop_callback = EarlyStopping(
+    monitor="val_loss",
+    patience=4,
+    mode="min",
+)
+
+
+def load_image(image_path, transform=None):
+    image = Image.open(image_path).convert("RGB")
+
+    if transform:
+        image = transform(image)
+
+    return image
+
+
+def predict_single_image(image_path, model, transform=None):
+    image = load_image(image_path, transform)
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    model.to(device)
+
+    image = image.to(device)
+
+    model.eval()
+
+    with torch.no_grad():
+        image = image.unsqueeze(0)
+        output = model(image).squeeze()
+        print(output)
+        prediction = torch.sigmoid(output).item()
+
+    return prediction
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--ckpt_path",
+    help="checkpoint to continue from",
+    required=False,
+)
+parser.add_argument(
+    "--predict",
+    help="predict on test set",
+    action="store_true",
+)
+parser.add_argument("--reset", help="reset training", action="store_true")
+parser.add_argument(
+    "--predict_image",
+    help="predict the class of a single image",
+    action="store_true",
+)
+parser.add_argument(
+    "--image_path",
+    help="path to the image to predict",
+    type=str,
+    required=False,
+)
+parser.add_argument(
+    "--dir",
+    help="path to the images to predict",
+    type=str,
+    required=False,
+)
+parser.add_argument(
+    "--output_file",
+    help="path to output file",
+    type=str,
+    required=False,
+)
+args = parser.parse_args()
+
+train_domains = [0, 1, 4]
+val_domains = [0, 1, 4]
+lmd_value = 0
+
+if args.predict:
+    test_dl = load_dataloader(
+        [0, 1, 2, 3, 4],
+        "test",
+        batch_size=128,
+        num_workers=1,
+    )
+    model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
+    trainer = pl.Trainer()
+    predictions = trainer.predict(model, dataloaders=test_dl)
+    preds, labels, domains = zip(*predictions)
+    preds = torch.cat(preds).cpu().numpy()
+    labels = torch.cat(labels).cpu().numpy()
+    domains = torch.cat(domains).cpu().numpy()
+    print(preds.shape, labels.shape, domains.shape)
+    df = pd.DataFrame({"preds": preds, "labels": labels, "domains": domains})
+    filename = "preds-" + args.ckpt_path.split("/")[-1]
+    df.to_csv(f"outputs/{filename}.csv", index=False)
+elif args.predict_image:
+    image_path = args.image_path
+    model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
+
+    # Define the transformations for the image
+    # transform = transforms.Compose(
+    #     [
+    #         transforms.Resize((224, 224)),  # Image size expected by ResNet50
+    #         transforms.ToTensor(),
+    #         transforms.Normalize(
+    #             mean=[0.485, 0.456, 0.406],
+    #             std=[0.229, 0.224, 0.225],
+    #         ),
+    #     ],
+    # )
+
+    transform = transforms.Compose(
+        [
+            transforms.CenterCrop((256, 256)),
+            transforms.ToTensor(),
+        ],
+    )
+
+    prediction = predict_single_image(image_path, model, transform)
+    print("prediction", prediction)
+
+    # Output the prediction
+    print(
+        f"Prediction for {image_path}: "
+        f"{'Human' if prediction <= 0.001 else 'Generated'}",
+    )
+elif args.dir is not None:
+    predictions = []
+    model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
+    # Define the transformations for the image
+    # transform = transforms.Compose(
+    #     [
+    #         transforms.Resize((224, 224)),  # Image size expected by ResNet50
+    #         transforms.ToTensor(),
+    #         transforms.Normalize(
+    #             mean=[0.485, 0.456, 0.406],
+    #             std=[0.229, 0.224, 0.225],
+    #         ),
+    #     ],
+    # )
+    transform = transforms.Compose(
+        [
+            transforms.CenterCrop((256, 256)),
+            transforms.ToTensor(),
+        ],
+    )
+    for root, dirs, files in os.walk(os.path.abspath(args.dir)):
+        for f_name in files:
+            f = os.path.join(root, f_name)
+            print(f"Predicting: {f}")
+            p = predict_single_image(f, model, transform)
+            predictions.append([f, f.split("/")[-2], p, p > 0.5])
+            print(f"--predicted: {p}")
+
+    df = pd.DataFrame(predictions, columns=["path", "folder", "pred", "class"])
+    df.to_csv(args.output_file, index=False)
+else:
+    train_dl = load_dataloader(
+        train_domains,
+        "train",
+        batch_size=128,
+        num_workers=4,
+    )
+    logging.info("Training dataloader loaded")
+    val_dl = load_dataloader(val_domains, "val", batch_size=128, num_workers=4)
+    logging.info("Validation dataloader loaded")
+
+    if args.reset:
+        model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
+    else:
+        model = ImageClassifier(lmd=lmd_value)
+    trainer = pl.Trainer(
+        callbacks=[checkpoint_callback, early_stop_callback],
+        max_steps=20000,
+        val_check_interval=1000,
+        check_val_every_n_epoch=None,
+    )
+    trainer.fit(
+        model=model,
+        train_dataloaders=train_dl,
+        val_dataloaders=val_dl,
+        ckpt_path=args.ckpt_path if not args.reset else None,
+    )

src/images/Diffusion/sample_laion_script.ipynb

@@ -0,0 +1,73 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import dask.dataframe as dd\n",
+    "from dask.diagnostics import ProgressBar\n",
+    "import os\n",
+    "\n",
+    "directory_path = '/Users/fionachow/Documents/NYU/CDS/Fall 2023/CSCI - GA 2271 - Computer Vision/Project/'\n",
+    "\n",
+    "file_prefix = 'part'\n",
+    "\n",
+    "def list_files_with_prefix(directory, prefix):\n",
+    "    file_paths = []\n",
+    "\n",
+    "    for root, _, files in os.walk(directory):\n",
+    "        for file in files:\n",
+    "            if file.startswith(prefix):\n",
+    "                absolute_path = os.path.join(root, file)\n",
+    "                file_paths.append(absolute_path)\n",
+    "\n",
+    "    return file_paths\n",
+    "\n",
+    "laion_file_paths = list_files_with_prefix(directory_path, file_prefix)\n",
+    "\n",
+    "dataframes = [dd.read_parquet(file) for file in laion_file_paths]\n",
+    "combined_dataframe = dd.multi.concat(dataframes)\n",
+    "\n",
+    "with ProgressBar():\n",
+    "    row_count = combined_dataframe.shape[0].compute()\n",
+    "    print(row_count)\n",
+    "\n",
+    "filtered_df = combined_dataframe[combined_dataframe['NSFW'] == \"UNLIKELY\"]\n",
+    "\n",
+    "num_samples = 225_000\n",
+    "selected_rows = filtered_df.sample(frac=num_samples / filtered_df.shape[0].compute())\n",
+    "\n",
+    "with ProgressBar():\n",
+    "    sampled_rows = selected_rows.compute()\n",
+    "\n",
+    "print(len(sampled_rows))\n",
+    "\n",
+    "with ProgressBar():\n",
+    "    selected_rows.to_parquet('laion_sampled', write_index=False)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "bloom",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/images/Diffusion/scrape.py

@@ -0,0 +1,149 @@
+import argparse
+import time
+
+import polars as pl
+import requests
+
+
+def call_api(param):
+    url = "https://api.pullpush.io/reddit/search/submission/"
+    response = requests.get(url, params=param)
+    json_data = response.json()["data"]
+    create_utc = []
+    media_id = []
+    media_type_ls = []
+    post_ids = []
+    post_titles = []
+    cur_utc = 0
+    for submission in json_data:
+        cur_flair = submission["link_flair_text"]
+        cur_utc = submission["created_utc"]
+        media_ls = (
+            submission["media_metadata"]
+            if "media_metadata" in submission.keys()
+            else None
+        )
+        if param["flair"] is not None and cur_flair != param["flair"]:
+            continue
+        if media_ls is None:
+            continue
+        for id in media_ls.keys():
+            if media_ls[id]["status"] != "valid":
+                continue
+            try:
+                media_type = media_ls[id]["m"]
+            except:  # noqa
+                # video will error out
+                continue
+            if media_type == "image/png":
+                media_type_ls.append("png")
+            elif media_type == "image/jpg":
+                media_type_ls.append("jpg")
+            else:
+                continue
+            create_utc.append(int(cur_utc))
+            post_ids.append(submission["id"])
+            post_titles.append(submission["title"])
+            media_id.append(id)
+
+    df = pl.DataFrame(
+        {
+            "create_utc": create_utc,
+            "media_id": media_id,
+            "media_type": media_type_ls,
+            "post_id": post_ids,
+            "post_title": post_titles,
+        },
+        schema={
+            "create_utc": pl.Int64,
+            "media_id": pl.Utf8,
+            "media_type": pl.Utf8,
+            "post_id": pl.Utf8,
+            "post_title": pl.Utf8,
+        },
+    )
+    return df, int(cur_utc)
+
+
+def scraping_loop(
+    subreddit,
+    flair,
+    max_num=30000,
+    output_name=None,
+    before=None,
+):
+    collected_all = []
+    collected_len = 0
+    last_timestamp = int(time.time()) if before is None else before
+    param = {
+        "subreddit": subreddit,
+        "flair": flair,
+        "before": last_timestamp,
+    }
+    while collected_len < max_num:
+        collected_df, last_timestamp = call_api(param)
+        if collected_df.shape[0] == 0:
+            print("No more data, saving current data and exiting...")
+            break
+        collected_all.append(collected_df)
+        collected_len += collected_df.shape[0]
+        print(
+            f"collected_len: {collected_len}, "
+            f"last_timestamp: {last_timestamp}",
+        )
+        param["before"] = last_timestamp
+
+    df = pl.concat(collected_all)
+    df = (
+        df.with_columns(
+            pl.col("media_id")
+            .str.replace(r"^", "https://i.redd.it/")
+            .alias("url1"),
+            pl.col("create_utc")
+            .cast(pl.Int64)
+            .cast(pl.Utf8)
+            .str.to_datetime("%s")
+            .alias("time"),
+        )
+        .with_columns(
+            pl.col("media_type").str.replace(r"^", ".").alias("url2"),
+        )
+        .with_columns(
+            pl.concat_str(
+                [pl.col("url1"), pl.col("url2")],
+                separator="",
+            ).alias("url"),
+        )
+        .select("time", "url", "post_id", "post_title")
+    )
+    if output_name is None:
+        output_name = subreddit
+    df.write_parquet(f"urls/{output_name}.parquet")
+    df.select("url").write_csv(f"urls/{output_name}.csv", has_header=False)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--subreddit", help="subreddit name")
+    parser.add_argument("--flair", help="flair filter", default=None, type=str)
+    parser.add_argument(
+        "--max_num",
+        help="max number of posts to scrape",
+        default=30000,
+        type=int,
+    )
+    parser.add_argument(
+        "--output_name",
+        help="custom output name",
+        default=None,
+    )
+    parser.add_argument(
+        "--before",
+        help="before timestamp",
+        default=None,
+        type=int,
+    )
+
+    args = parser.parse_args()
+
+    scraping_loop(**args.__dict__)

src/images/Diffusion/utils_sampling.py

@@ -0,0 +1,94 @@
+import collections
+import random
+from typing import Callable
+
+from torchdata.datapipes.iter import IterDataPipe
+
+
+def get_second_entry(sample):
+    return sample[1]
+
+
+class UnderSamplerIterDataPipe(IterDataPipe):
+    """Dataset wrapper for under-sampling.
+
+    Copied from: https://github.com/MaxHalford/pytorch-resample/blob/master/pytorch_resample/under.py # noqa
+    Modified to work with multiple labels.
+
+    MIT License
+
+    Copyright (c) 2020 Max Halford
+
+    This method is based on rejection sampling.
+
+    Parameters:
+        dataset
+        desired_dist: The desired class distribution.
+            The keys are the classes whilst the
+            values are the desired class percentages.
+            The values are normalised so that sum up
+            to 1.
+        label_getter: A function that takes a sample and returns its label.
+        seed: Random seed for reproducibility.
+
+    Attributes:
+        actual_dist: The counts of the observed sample labels.
+        rng: A random number generator instance.
+
+    References:
+        - https://www.wikiwand.com/en/Rejection_sampling
+
+    """
+
+    def __init__(
+        self,
+        dataset: IterDataPipe,
+        desired_dist: dict,
+        label_getter: Callable = get_second_entry,
+        seed: int = None,
+    ):
+
+        self.dataset = dataset
+        self.desired_dist = {
+            c: p / sum(desired_dist.values()) for c, p in desired_dist.items()
+        }
+        self.label_getter = label_getter
+        self.seed = seed
+
+        self.actual_dist = collections.Counter()
+        self.rng = random.Random(seed)
+        self._pivot = None
+
+    def __iter__(self):
+
+        for dp in self.dataset:
+            y = self.label_getter(dp)
+
+            self.actual_dist[y] += 1
+
+            # To ease notation
+            f = self.desired_dist
+            g = self.actual_dist
+
+            # Check if the pivot needs to be changed
+            if y != self._pivot:
+                self._pivot = max(g.keys(), key=lambda y: f[y] / g[y])
+            else:
+                yield dp
+                continue
+
+            # Determine the sampling ratio if the observed label
+            # is not the pivot
+            M = f[self._pivot] / g[self._pivot]
+            ratio = f[y] / (M * g[y])
+
+            if ratio < 1 and self.rng.random() < ratio:
+                yield dp
+
+    @classmethod
+    def expected_size(cls, n, desired_dist, actual_dist):
+        M = max(
+            desired_dist.get(k) / actual_dist.get(k)
+            for k in set(desired_dist) | set(actual_dist)
+        )
+        return int(n / M)

src/images/Diffusion/visualizations.ipynb

@@ -0,0 +1,196 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install polars-lts-cpu"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import polars as pl\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score, f1_score"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def pfbeta(labels, predictions, beta=1):\n",
+    "    y_true_count = 0\n",
+    "    ctp = 0\n",
+    "    cfp = 0\n",
+    "\n",
+    "    for idx in range(len(labels)):\n",
+    "        prediction = min(max(predictions[idx], 0), 1)\n",
+    "        if (labels[idx]):\n",
+    "            y_true_count += 1\n",
+    "            ctp += prediction\n",
+    "        else:\n",
+    "            cfp += prediction\n",
+    "\n",
+    "    beta_squared = beta * beta\n",
+    "    c_precision = ctp / (ctp + cfp)\n",
+    "    c_recall = ctp / y_true_count\n",
+    "    if (c_precision > 0 and c_recall > 0):\n",
+    "        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n",
+    "        return result\n",
+    "    else:\n",
+    "        return 0\n",
+    "\n",
+    "def get_part_metrics(df: pl.DataFrame, threshold=0.3) -> dict:\n",
+    "    df = df.with_columns((df[\"preds\"] > threshold).alias(\"preds_bin\"))\n",
+    "    metrics = {}\n",
+    "    # binary metrics using the threshold\n",
+    "    metrics[\"accuracy\"] = accuracy_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
+    "    metrics[\"precision\"] = precision_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
+    "    metrics[\"recall\"] = recall_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
+    "    metrics[\"f1\"] = f1_score(df[\"labels\"].to_numpy(), df[\"preds_bin\"].to_numpy())\n",
+    "    # probabilistic F1 (doesn't depend on the threshold)\n",
+    "    metrics[\"pf1\"] = pfbeta(df[\"labels\"].to_numpy(), df[\"preds\"].to_numpy())\n",
+    "    # ROC AUC\n",
+    "    metrics[\"roc_auc\"] = roc_auc_score(df[\"labels\"].to_numpy(), df[\"preds\"].to_numpy())\n",
+    "    return metrics\n",
+    "\n",
+    "\n",
+    "def get_all_metrics(df: pl.DataFrame, threshold=0.3) -> pd.DataFrame:\n",
+    "    groups = [list(range(5)), [0, 1], [0, 4], [0, 2], [0, 3]]\n",
+    "    group_names = [\"all\", \"StableDiffusion\", \"Midjourney\", \"Dalle2\", \"Dalle3\"]\n",
+    "    all_metrics = []\n",
+    "    for i, g in enumerate(groups):\n",
+    "        subset = df.filter(pl.col(\"domains\").is_in(g))\n",
+    "        metrics = get_part_metrics(subset, threshold=threshold)\n",
+    "        metrics[\"group\"] = group_names[i]\n",
+    "        all_metrics.append(metrics)\n",
+    "    \n",
+    "    return pd.DataFrame(all_metrics)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load the data from the output files\n",
+    "df1 = pl.read_csv('/Users/fionachow/Downloads/outputs/preds-image-classifier-1.csv')\n",
+    "df14 = pl.read_csv('/Users/fionachow/Downloads/outputs/preds-image-classifier-14.csv')\n",
+    "df142 = pl.read_csv('/Users/fionachow/Downloads/outputs/preds-image-classifier-142.csv')\n",
+    "df1423 = pl.read_csv('/Users/fionachow/Downloads/outputs/preds-image-classifier-1423.csv')\n",
+    "\n",
+    "metrics_df1 = get_all_metrics(df1, threshold=0.5)\n",
+    "metrics_df14 = get_all_metrics(df14, threshold=0.5)\n",
+    "metrics_df142 = get_all_metrics(df142, threshold=0.5)\n",
+    "metrics_df1423 = get_all_metrics(df1423, threshold=0.5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "metrics_df1.info()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sns.set()\n",
+    "\n",
+    "models = ['StableDiffusion', 'Midjourney', 'Dalle2', 'Dalle3']\n",
+    "metrics = ['accuracy', 'f1', 'pf1', 'roc_auc']\n",
+    "\n",
+    "file_map = {\n",
+    "    ('StableDiffusion',): metrics_df1,\n",
+    "    ('StableDiffusion', 'Midjourney'): metrics_df14,\n",
+    "    ('StableDiffusion', 'Midjourney', 'Dalle2'): metrics_df142,\n",
+    "    ('StableDiffusion', 'Midjourney', 'Dalle2', 'Dalle3'): metrics_df1423,\n",
+    "}\n",
+    "\n",
+    "def create_heatmap_data(metric):\n",
+    "    data = pd.DataFrame(index=models[::-1], columns=models)\n",
+    "    for i, model_x in enumerate(models):\n",
+    "        for j, model_y in enumerate(models[::-1]):\n",
+    "            \n",
+    "            if i == 0:\n",
+    "                relevant_df = metrics_df1\n",
+    "            elif i == 1:\n",
+    "                relevant_df = metrics_df14\n",
+    "            elif i == 2:\n",
+    "                relevant_df = metrics_df142\n",
+    "            else:\n",
+    "                relevant_df = metrics_df1423\n",
+    "\n",
+    "            # Debugging: print the DataFrame being used and the model_y\n",
+    "            #print(f\"Using DataFrame for {models[:i+1]}, model_y: {model_y}\")\n",
+    "\n",
+    "            # Extract the metric value\n",
+    "            if model_y in relevant_df['group'].values:\n",
+    "                metric_value = relevant_df[relevant_df['group'] == model_y][metric].values[0]\n",
+    "                # Debugging: print the extracted metric value\n",
+    "                #print(f\"Metric value for {model_y}: {metric_value}\")\n",
+    "            else:\n",
+    "                metric_value = float('nan')  # Handle non-existent cases\n",
+    "                # Debugging: print a message for non-existent cases\n",
+    "                #print(f\"No data for combination: {model_x}, {model_y}\")\n",
+    "\n",
+    "            data.at[model_y, model_x] = metric_value\n",
+    "    \n",
+    "    for col in data.columns:\n",
+    "        data[col] = pd.to_numeric(data[col], errors='coerce')\n",
+    "\n",
+    "    # Debugging: print the final DataFrame\n",
+    "    # print(f\"Final Data for metric {metric}:\")\n",
+    "    # print(data)\n",
+    "    # print(data.dtypes)\n",
+    "    return data\n",
+    "\n",
+    "for metric in metrics:\n",
+    "    plt.figure(figsize=(10, 8))\n",
+    "    sns.heatmap(create_heatmap_data(metric), annot=True, cmap='coolwarm', fmt='.3f')\n",
+    "    plt.title(f\"Heatmap for {metric}\")\n",
+    "    plt.xlabel(\"Trained On (x-axis)\")\n",
+    "    plt.ylabel(\"Tested On (y-axis)\")\n",
+    "    plt.show()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "bloom",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/images/README.md

@@ -0,0 +1,64 @@
+# AI-generated image detection
+**(Work In Progress)**
+
+- [ ] Refactor code
+- [ ] Review dependencies
+- [ ] Containerize (Docker)
+- [ ] Update documentation
+
+## AI-Generated Image detection
+
+This part handles the detection of AI-generated images.
+The current code contains two classifiers to detect AI-generated images from two types of architectures:
+- GANs
+
+## Model weights
+
+### 1. CNN Detection
+
+Run the `download_weights_CNN.sh` script:
+
+```commandline
+bash download_weights_CNN.sh
+```
+
+Note: you need `wget` installed on your system (it is by default for most Linux systems).
+
+### 2. Diffusion
+
+**TODO**
+
+
+## Run the models
+
+Make sure you have the weights available before doing so.
+
+**TODO: environments**
+
+### 1. CNN Detection
+
+```commandline
+python CNN_model_classifier.py
+```
+Available options:
+
+- `-f / --file` (default=`'examples_realfakedir'`)
+- `-m / --model_path` (default=`'weights/blur_jpg_prob0.5.pth'`)
+- `-c / --crop` (default=`None`):  Specify crop size (int) by default, do not crop.
+- `--use_cpu`: use cpu (by default uses GPU) -> **TODO: remove (obsolete)**
+
+Example usage:
+
+```commandline
+python CNN_model_classifier.py -f examples/real.png -m weights/blur_jpg_prob0.5.pth
+```
+
+### 2. Diffusion detection
+
+**TODO**
+
+## References
+
+Based on:
+- https://github.com/hoangthuc701/GenAI-image-detection
+- https://github.com/ptmaimai106/DetectGenerateImageByRealImageOnly

src/images/Search_Image/Bing_search.py

@@ -0,0 +1,93 @@
+import json
+import os
+from dotenv import load_dotenv
+import requests
+
+# Load Bing Search API key
+load_dotenv() 
+BING_API_KEY = os.getenv("BING_API_KEY")
+
+def print_json(obj):
+    """Print the object as json"""
+    print(json.dumps(obj, sort_keys=True, indent=4, separators=(',', ': ')))
+
+
+def get_image_urls(search_results):
+    """
+    Extracts image URLs from Bing Visual Search response.
+    Ref: https://learn.microsoft.com/en-us/bing/search-apis/bing-visual-search/how-to/search-response
+
+    Args:
+        search_results: A dict containing the Bing VisualSearch response data.
+
+    Returns:
+        A tuple containing two lists:
+            - List of image URLs from "PagesIncluding" section.
+            - List of image URLs from "VisualSearch" section (backup).
+    """
+    
+    pages_including_urls = []
+    visual_search_urls = []
+    
+    if "tags" not in search_results:
+        return pages_including_urls, visual_search_urls
+    
+    # Check for required keys directly
+    if not any(action.get("actions") for action in search_results["tags"]):
+        return pages_including_urls, visual_search_urls
+    
+    
+    for action in search_results["tags"]:
+        for result in action.get("actions", []):
+            # actions = PagesIncluding, main results
+            if result["name"] == "PagesIncluding":
+                pages_including_urls.extend(item["contentUrl"] for item in result["data"]["value"])
+            # actions = VisualSearch, back up results 
+            elif result["name"] == "VisualSearch":
+                visual_search_urls.extend(item["contentUrl"] for item in result["data"]["value"])
+
+    return pages_including_urls, visual_search_urls
+
+def reverse_image_search(image_path, subscription_key=BING_API_KEY):
+    """Performs a reverse image search using the Bing Visual Search API.
+
+    Args:
+        image_path: The path to the image file to search for.
+
+    Returns:
+        A list of image URLs found that are similar to the image in the 
+            specified path.
+
+    Raises:
+        requests.exceptions.RequestException: If the API request fails.
+    """
+    base_uri = "https://api.bing.microsoft.com/v7.0/images/visualsearch"
+    headers = {"Ocp-Apim-Subscription-Key": subscription_key}
+    
+    try:
+        files = {"image": ("image", open(image_path, "rb"))}
+        response = requests.post(base_uri, headers=headers, files=files)
+        response.raise_for_status()
+        search_results = response.json()
+        
+        return search_results
+
+    except requests.exceptions.RequestException as e:
+        raise requests.exceptions.RequestException(f"API request failed: {e}")
+    except OSError as e:
+        raise OSError(f"Error opening image file: {e}")
+
+if __name__ == "__main__":
+    # Example usage:
+    image_path = "data/test_data/human_news.jpg"
+    try:
+        search_results = reverse_image_search(image_path)
+        image_urls, backup_image_urls = get_image_urls(search_results)
+
+        # Print the results
+        print("Image URLs from PagesIncluding:")
+        print(image_urls)
+        print("\nImage URLs from VisualSearch (backup):")
+        print(backup_image_urls)
+    except Exception as e:
+        print(f"An error occurred: {e}")

src/images/Search_Image/image_model_share.py

@@ -0,0 +1,142 @@
+from sklearn.metrics import roc_auc_score
+from torchmetrics import Accuracy, Recall
+import pytorch_lightning as pl
+import timm  
+import torch
+from pytorch_lightning.callbacks import Model, EarlyStopping
+import logging
+from PIL import Image
+import torchvision.transforms as transforms
+from torchvision.transforms import v2
+
+logging.basicConfig(filename='training.log',filemode='w',level=logging.INFO, force=True)
+CHECKPOINT = "models/image_classifier/image-classifier-step=8008-val_loss=0.11.ckpt"
+
+
+
+class ImageClassifier(pl.LightningModule):
+    def __init__(self, lmd=0):
+        super().__init__()
+        self.model = timm.create_model('resnet50', pretrained=True, num_classes=1)
+        self.accuracy = Accuracy(task='binary', threshold=0.5)
+        self.recall = Recall(task='binary', threshold=0.5)  
+        self.validation_outputs = []
+        self.lmd = lmd
+
+    def forward(self, x):
+        return self.model(x)
+
+    def training_step(self, batch):
+        images, labels, _ = batch
+        outputs = self.forward(images).squeeze()
+        
+        print(f"Shape of outputs (training): {outputs.shape}")
+        print(f"Shape of labels (training): {labels.shape}")
+        
+        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
+        logging.info(f"Training Step - ERM loss: {loss.item()}")
+        loss += self.lmd * (outputs ** 2).mean() # SD loss penalty
+        logging.info(f"Training Step - SD loss: {loss.item()}")
+        return loss
+
+    def validation_step(self, batch):
+        images, labels, _ = batch
+        outputs = self.forward(images).squeeze()
+
+        if outputs.shape == torch.Size([]):
+            return
+        
+        print(f"Shape of outputs (validation): {outputs.shape}")
+        print(f"Shape of labels (validation): {labels.shape}")
+
+        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
+        preds = torch.sigmoid(outputs)
+        self.log('val_loss', loss, prog_bar=True, sync_dist=True)
+        self.log('val_acc', self.accuracy(preds, labels.int()), prog_bar=True, sync_dist=True)
+        self.log('val_recall', self.recall(preds, labels.int()), prog_bar=True, sync_dist=True)
+        output = {"val_loss": loss, "preds": preds, "labels": labels}
+        self.validation_outputs.append(output)
+        logging.info(f"Validation Step - Batch loss: {loss.item()}")
+        return output
+    
+    def predict_step(self, batch):
+        images, label, domain = batch
+        outputs = self.forward(images).squeeze()
+        preds = torch.sigmoid(outputs)
+        return preds, label, domain
+
+    def on_validation_epoch_end(self):
+        if not self.validation_outputs:
+            logging.warning("No outputs in validation step to process")
+            return
+        preds = torch.cat([x['preds'] for x in self.validation_outputs])
+        labels = torch.cat([x['labels'] for x in self.validation_outputs])
+        if labels.unique().size(0) == 1:
+            logging.warning("Only one class in validation step")
+            return
+        auc_score = roc_auc_score(labels.cpu(), preds.cpu())
+        self.log('val_auc', auc_score, prog_bar=True, sync_dist=True)
+        logging.info(f"Validation Epoch End - AUC score: {auc_score}")
+        self.validation_outputs = []
+
+    def configure_optimizers(self):
+        optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0005)
+        return optimizer
+
+
+
+def load_image(image_path, transform=None):
+    image = Image.open(image_path).convert('RGB')
+    
+    if transform:
+        image = transform(image)
+    
+    return image
+
+
+def predict_single_image(image_path, model, transform=None):
+    image = load_image(image_path, transform) 
+    
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    
+    model.to(device)
+    
+    image = image.to(device)
+
+    model.eval()
+    
+    with torch.no_grad():
+        image = image.unsqueeze(0)  
+        output = model(image).squeeze() 
+        print(output)
+        prediction = torch.sigmoid(output).item()  
+    
+    return prediction
+
+
+def image_generation_detection(image_path):
+    model = ImageClassifier.load_from_checkpoint(CHECKPOINT)
+
+    transform = v2.Compose([
+        transforms.ToTensor(),
+        v2.CenterCrop((256, 256)),
+    ])    
+
+    prediction = predict_single_image(image_path, model, transform) 
+    print("prediction",prediction)
+    
+    result = ""
+    if prediction <= 0.2:
+        result += "Most likely human"
+        image_prediction_label = "HUMAN"
+    else:
+        result += "Most likely machine"
+        image_prediction_label = "MACHINE"
+    image_confidence = min(1, 0.5 + abs(prediction - 0.2))
+    result += f" with confidence = {round(image_confidence * 100, 2)}%"
+    image_confidence = round(image_confidence * 100, 2)
+    return image_prediction_label, image_confidence
+
+
+if __name__ == "__main__":  
+    pass

src/images/Search_Image/search.py

@@ -0,0 +1,56 @@
+from google_img_source_search import ReverseImageSearcher
+import requests
+from io import BytesIO
+from PIL import Image
+import imagehash
+from google_img_source_search import ReverseImageSearcher
+
+def get_image_from_url(url):
+    response = requests.get(url)
+    return Image.open(BytesIO(response.content))
+
+def standardize_image(image):
+    # Convert to RGB if needed
+    if image.mode in ('RGBA', 'LA'):
+        background = Image.new('RGB', image.size, (255, 255, 255))
+        background.paste(image, mask=image.split()[-1])
+        image = background
+    elif image.mode != 'RGB':
+        image = image.convert('RGB')
+    
+    # Resize to standard size (e.g. 256x256)
+    standard_size = (256, 256)
+    image = image.resize(standard_size)
+    
+    return image
+
+def compare_images(image1, image2):
+    # Standardize both images first
+    img1_std = standardize_image(image1)
+    img2_std = standardize_image(image2)
+    
+    hash1 = imagehash.average_hash(img1_std)
+    hash2 = imagehash.average_hash(img2_std)
+    return hash1 - hash2  # Returns the Hamming distance between the hashes
+
+if __name__ == '__main__':
+    image_url = 'https://i.pinimg.com/originals/c4/50/35/c450352ac6ea8645ead206721673e8fb.png'
+    
+    # Get the image from URL
+    url_image = get_image_from_url(image_url)
+
+    # Search image
+    rev_img_searcher = ReverseImageSearcher()
+    res = rev_img_searcher.search(image_url)
+
+    for search_item in res:
+        print(f'Title: {search_item.page_title}')
+        # print(f'Site: {search_item.page_url}')
+        print(f'Img: {search_item.image_url}\n')
+
+        # Compare each search result image with the input image
+        result_image = get_image_from_url(search_item.image_url)
+        result_difference = compare_images(result_image, url_image)
+        print(f"Difference with search result: {result_difference}")
+        if result_difference == 0: 
+            break 

src/images/Search_Image/search_2.py

@@ -0,0 +1,150 @@
+import time
+import logging
+import requests
+from bs4 import BeautifulSoup
+from typing import Dict, Optional
+from urllib.parse import quote, urlparse
+
+logging.basicConfig(
+    filename='error.log',
+    level=logging.INFO,
+    format='%(asctime)s | [%(levelname)s]: %(message)s',
+    datefmt='%m-%d-%Y / %I:%M:%S %p'
+)
+
+class SearchResults:
+    def __init__(self, results):
+        self.results = results
+
+    def __str__(self):
+        output = ""
+        for result in self.results:
+            output += "---\n"
+            output += f"Title: {result.get('title', 'Title not found')}\n"
+            output += f"Link: {result.get('link', 'Link not found')}\n"
+            output += "---\n"
+        return output
+
+class GoogleReverseImageSearch:
+    def __init__(self):
+        self.base_url = "https://www.google.com/searchbyimage"
+        self.headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}
+        self.retry_count = 3
+        self.retry_delay = 1
+
+    def response(self, query: str, image_url: str, max_results: int = 10, delay: int = 1) -> SearchResults:
+        self._validate_input(query, image_url)
+        
+        encoded_query = quote(query)
+        encoded_image_url = quote(image_url)
+
+        url = f"{self.base_url}?q={encoded_query}&image_url={encoded_image_url}&sbisrc=cr_1_5_2"
+
+        all_results = []
+        start_index = 0
+
+        while len(all_results) < max_results:
+            if start_index != 0:
+                time.sleep(delay)
+                
+            paginated_url = f"{url}&start={start_index}"
+
+            response = self._make_request(paginated_url)
+            if response is None:
+                break
+
+            search_results, valid_content = self._parse_search_results(response.text)
+            if not valid_content:
+                logging.warning("Unexpected HTML structure encountered.")
+                break
+
+            for result in search_results:
+                if len(all_results) >= max_results:
+                    break
+                data = self._extract_result_data(result)
+                if data and data not in all_results:
+                    all_results.append(data)
+
+            start_index += (len(all_results)-start_index)
+
+        if len(all_results) == 0:
+            logging.warning(f"No results were found for the given query: [{query}], and/or image URL: [{image_url}].")
+            return "No results found. Please try again with a different query and/or image URL."
+        else:
+            return SearchResults(all_results[:max_results])
+    
+    def _validate_input(self, query: str, image_url: str):
+        if not query:
+            raise ValueError("Query not found. Please enter a query and try again.")
+        if not image_url:
+            raise ValueError("Image URL not found. Please enter an image URL and try again.")
+        if not self._validate_image_url(image_url):
+            raise ValueError("Invalid image URL. Please enter a valid image URL and try again.")
+    
+    def _validate_image_url(self, url: str) -> bool:
+        parsed_url = urlparse(url)
+        path = parsed_url.path.lower()
+        valid_extensions = (".jpg", ".jpeg", ".png", ".webp")
+        return any(path.endswith(ext) for ext in valid_extensions)
+    
+    def _make_request(self, url: str):
+        attempts = 0
+        while attempts < self.retry_count:
+            try:
+                response = requests.get(url, headers=self.headers)
+                if response.headers.get('Content-Type', '').startswith('text/html'):
+                    response.raise_for_status()
+                    return response
+                else:
+                    logging.warning("Non-HTML content received.")
+                    return None
+            except requests.exceptions.HTTPError as http_err:
+                logging.error(f"HTTP error occurred: {http_err}")
+                attempts += 1
+                time.sleep(self.retry_delay)
+            except Exception as err:
+                logging.error(f"An error occurred: {err}")
+                return None
+        return None
+
+    def _parse_search_results(self, html_content: str) -> (Optional[list], bool):
+        try:
+            soup = BeautifulSoup(html_content, "html.parser")
+            return soup.find_all('div', class_='g'), True
+        except Exception as e:
+            logging.error(f"Error parsing HTML content: {e}")
+            return None, False
+
+    def _extract_result_data(self, result) -> Dict:
+        link = result.find('a', href=True)['href'] if result.find('a', href=True) else None
+        title = result.find('h3').get_text(strip=True) if result.find('h3') else None
+        return {"link": link, "title": title} if link and title else {}
+    
+
+if __name__ == "__main__":
+    # request = GoogleReverseImageSearch()
+
+    # response = request.response(
+    #     query="Example Query",
+    #     image_url="https://ichef.bbci.co.uk/images/ic/1024xn/p0khzhhl.jpg.webp",
+    #     max_results=5
+    # )
+
+    # print(response)
+
+    # Path to local image
+    image_path = "data/test_data/towel.jpg"
+    image_path = "C:\\TTProjects\\prj-nict-ai-content-detection\\data\\test_data\\towel.jpg"
+
+    import json
+    file_path = image_path
+    search_url = 'https://yandex.ru/images/search'
+    files = {'upfile': ('blob', open(file_path, 'rb'), 'image/jpeg')}
+    params = {'rpt': 'imageview', 'format': 'json', 'request': '{"blocks":[{"block":"b-page_type_search-by-image__link"}]}'}
+    response = requests.post(search_url, params=params, files=files)
+    query_string = json.loads(response.content)['blocks'][0]['params']['url']
+    img_search_url = search_url + '?' + query_string
+    print(img_search_url)
+
+    response = requests.get(img_search_url)
+    print(response.text)

src/images/Search_Image/search_yandex.py

@@ -0,0 +1,177 @@
+import time
+import logging
+import requests
+from bs4 import BeautifulSoup
+from typing import Dict, Optional
+from urllib.parse import quote, urlparse
+
+logging.basicConfig(
+    filename='error.log',
+    level=logging.INFO,
+    format='%(asctime)s | [%(levelname)s]: %(message)s',
+    datefmt='%m-%d-%Y / %I:%M:%S %p'
+)
+
+class SearchResults:
+    def __init__(self, results):
+        self.results = results
+
+    def __str__(self):
+        output = ""
+        for result in self.results:
+            output += "---\n"
+            output += f"Title: {result.get('title', 'Title not found')}\n"
+            output += f"Link: {result.get('link', 'Link not found')}\n"
+            output += "---\n"
+        return output
+
+class ReverseImageSearch:
+    def __init__(self):
+        self.base_url = "https://yandex.ru/images/search"
+        self.headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}
+        self.retry_count = 3
+        self.retry_delay = 1
+
+    def response(self, query: str, image_url: str, max_results: int = 10, delay: int = 1) -> SearchResults:
+        self._validate_input(query, image_url)
+        
+        encoded_query = quote(query)
+        encoded_image_url = quote(image_url)
+
+        url = f"{self.base_url}?q={encoded_query}&image_url={encoded_image_url}&sbisrc=cr_1_5_2"
+
+        all_results = []
+        start_index = 0
+
+        while len(all_results) < max_results:
+            if start_index != 0:
+                time.sleep(delay)
+                
+            paginated_url = f"{url}&start={start_index}"
+
+            response = self._make_request(paginated_url)
+            if response is None:
+                break
+
+            search_results, valid_content = self._parse_search_results(response.text)
+            if not valid_content:
+                logging.warning("Unexpected HTML structure encountered.")
+                break
+
+            for result in search_results:
+                if len(all_results) >= max_results:
+                    break
+                data = self._extract_result_data(result)
+                if data and data not in all_results:
+                    all_results.append(data)
+
+            start_index += (len(all_results)-start_index)
+
+        if len(all_results) == 0:
+            logging.warning(f"No results were found for the given query: [{query}], and/or image URL: [{image_url}].")
+            return "No results found. Please try again with a different query and/or image URL."
+        else:
+            return SearchResults(all_results[:max_results])
+    
+    def _validate_input(self, query: str, image_url: str):
+        if not query:
+            raise ValueError("Query not found. Please enter a query and try again.")
+        if not image_url:
+            raise ValueError("Image URL not found. Please enter an image URL and try again.")
+        if not self._validate_image_url(image_url):
+            raise ValueError("Invalid image URL. Please enter a valid image URL and try again.")
+    
+    def _validate_image_url(self, url: str) -> bool:
+        parsed_url = urlparse(url)
+        path = parsed_url.path.lower()
+        valid_extensions = (".jpg", ".jpeg", ".png", ".webp")
+        return any(path.endswith(ext) for ext in valid_extensions)
+    
+    def _make_request(self, url: str):
+        attempts = 0
+        while attempts < self.retry_count:
+            try:
+                response = requests.get(url, headers=self.headers)
+                if response.headers.get('Content-Type', '').startswith('text/html'):
+                    response.raise_for_status()
+                    return response
+                else:
+                    logging.warning("Non-HTML content received.")
+                    return None
+            except requests.exceptions.HTTPError as http_err:
+                logging.error(f"HTTP error occurred: {http_err}")
+                attempts += 1
+                time.sleep(self.retry_delay)
+            except Exception as err:
+                logging.error(f"An error occurred: {err}")
+                return None
+        return None
+
+    def _parse_search_results(self, html_content: str) -> (Optional[list], bool):
+        try:
+            soup = BeautifulSoup(html_content, "html.parser")
+            return soup.find_all('div', class_='g'), True
+        except Exception as e:
+            logging.error(f"Error parsing HTML content: {e}")
+            return None, False
+
+    def _extract_result_data(self, result) -> Dict:
+        link = result.find('a', href=True)['href'] if result.find('a', href=True) else None
+        title = result.find('h3').get_text(strip=True) if result.find('h3') else None
+        return {"link": link, "title": title} if link and title else {}
+    
+def yandex_reverse_image_search(image_url):
+    # Simulate a user agent to avoid being blocked
+    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
+
+    try:
+        response = requests.get(image_url, headers=headers)
+        response.raise_for_status()  # Raise an exception for bad status codes
+
+        # Parse the HTML content
+        soup = BeautifulSoup(response.content, 'html.parser')
+
+        # Extract image URLs (example - adapt based on Yandex's HTML structure)
+        image_urls = [img['src'] for img in soup.find_all('img')]
+
+        # Extract related searches (example - adapt based on Yandex's HTML structure)
+        related_searches = [text for text in soup.find_all(class_="related-searches")] 
+
+        return image_urls, related_searches
+
+    except requests.exceptions.RequestException as e:
+        print(f"Error fetching image: {e}")
+        return [], []
+
+
+if __name__ == "__main__":
+    # request = GoogleReverseImageSearch()
+
+    # response = request.response(
+    #     query="Example Query",
+    #     image_url="https://ichef.bbci.co.uk/images/ic/1024xn/p0khzhhl.jpg.webp",
+    #     max_results=5
+    # )
+
+    # print(response)
+
+    # Path to local image
+    image_path = "data/test_data/towel.jpg"
+    image_path = "C:\\TTProjects\\prj-nict-ai-content-detection\\data\\test_data\\towel.jpg"
+
+    import json
+    file_path = image_path
+    search_url = 'https://yandex.ru/images/search'
+    files = {'upfile': ('blob', open(file_path, 'rb'), 'image/jpeg')}
+    params = {'rpt': 'imageview', 'format': 'json', 'request': '{"blocks":[{"block":"b-page_type_search-by-image__link"}]}'}
+    response = requests.post(search_url, params=params, files=files)
+    query_string = json.loads(response.content)['blocks'][0]['params']['url']
+    img_search_url = search_url + '?' + query_string
+    print(img_search_url)
+
+    image_urls, related_searches = yandex_reverse_image_search(img_search_url)
+
+    print("Image URLs:", image_urls)
+    print("Related Searches:", related_searches)
+
+    

src/images/diffusion_data_loader.py

@@ -0,0 +1,229 @@
+import argparse
+import collections
+import random
+from typing import Iterator
+
+import cv2
+import numpy as np
+import torchdata.datapipes as dp
+from imwatermark import WatermarkEncoder
+from PIL import (
+    Image,
+    ImageFile,
+)
+from torch.utils.data import DataLoader
+from torchdata.datapipes.iter import (
+    Concater,
+    FileLister,
+    FileOpener,
+    SampleMultiplexer,
+)
+from torchvision.transforms import v2
+from tqdm import tqdm
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+Image.MAX_IMAGE_PIXELS = 1000000000
+
+encoder = WatermarkEncoder()
+encoder.set_watermark("bytes", b"test")
+
+DOMAIN_LABELS = {
+    0: "laion",
+    1: "StableDiffusion",
+    2: "dalle2",
+    3: "dalle3",
+    4: "midjourney",
+}
+
+N_SAMPLES = {
+    0: (115346, 14418, 14419),
+    1: (22060, 2757, 2758),
+    4: (21096, 2637, 2637),
+    2: (13582, 1697, 1699),
+    3: (12027, 1503, 1504),
+}
+
+
+@dp.functional_datapipe("collect_from_workers")
+class WorkerResultCollector(dp.iter.IterDataPipe):
+    def __init__(self, source: dp.iter.IterDataPipe):
+        self.source = source
+
+    def __iter__(self) -> Iterator:
+        yield from self.source
+
+    def is_replicable(self) -> bool:
+        """Method to force data back to main process"""
+        return False
+
+
+def crop_bottom(image, cutoff=16):
+    return image[:, :-cutoff, :]
+
+
+def random_gaussian_blur(image, p=0.01):
+    if random.random() < p:
+        return v2.functional.gaussian_blur(image, kernel_size=5)
+    return image
+
+
+def random_invisible_watermark(image, p=0.2):
+    image_np = np.array(image)
+    image_np = np.transpose(image_np, (1, 2, 0))
+
+    if image_np.ndim == 2:  # Grayscale image
+        image_np = cv2.cvtColor(image_np, cv2.COLOR_GRAY2BGR)
+    elif image_np.shape[2] == 4:  # RGBA image
+        image_np = cv2.cvtColor(image_np, cv2.COLOR_RGBA2BGR)
+
+    if image_np.shape[0] < 256 or image_np.shape[1] < 256:
+        image_np = cv2.resize(
+            image_np,
+            (256, 256),
+            interpolation=cv2.INTER_AREA,
+        )
+
+    if random.random() < p:
+        return encoder.encode(image_np, method="dwtDct")
+
+    return image_np
+
+
+def build_transform(split: str):
+    train_transform = v2.Compose(
+        [
+            v2.Lambda(crop_bottom),
+            v2.RandomCrop((256, 256), pad_if_needed=True),
+            v2.Lambda(random_gaussian_blur),
+            v2.RandomGrayscale(p=0.05),
+            v2.Lambda(random_invisible_watermark),
+            v2.ToImage(),
+        ],
+    )
+
+    eval_transform = v2.Compose(
+        [
+            v2.CenterCrop((256, 256)),
+        ],
+    )
+    transform = train_transform if split == "train" else eval_transform
+
+    return transform
+
+
+def dp_to_tuple_train(input_dict):
+    transform = build_transform("train")
+    return (
+        transform(input_dict[".jpg"]),
+        input_dict[".label.cls"],
+        input_dict[".domain_label.cls"],
+    )
+
+
+def dp_to_tuple_eval(input_dict):
+    transform = build_transform("eval")
+    return (
+        transform(input_dict[".jpg"]),
+        input_dict[".label.cls"],
+        input_dict[".domain_label.cls"],
+    )
+
+
+def load_dataset(domains: list[int], split: str):
+    laion_lister = FileLister("./data/laion400m_data", f"{split}*.tar")
+    genai_lister = {
+        d: FileLister(
+            f"./data/genai-images/{DOMAIN_LABELS[d]}",
+            f"{split}*.tar",
+        )
+        for d in domains
+        if DOMAIN_LABELS[d] != "laion"
+    }
+    weight_genai = 1 / len(genai_lister)
+
+    def open_lister(lister):
+        opener = FileOpener(lister, mode="b")
+        return opener.load_from_tar().routed_decode().webdataset()
+
+    buffer_size1 = 100 if split == "train" else 10
+    buffer_size2 = 100 if split == "train" else 10
+
+    if split != "train":
+        all_lister = [laion_lister] + list(genai_lister.values())
+        dp = open_lister(Concater(*all_lister)).sharding_filter()
+    else:
+        laion_dp = (
+            open_lister(laion_lister.shuffle())
+            .cycle()
+            .sharding_filter()
+            .shuffle(buffer_size=buffer_size1)
+        )
+        genai_dp = {
+            open_lister(genai_lister[d].shuffle())
+            .cycle()
+            .sharding_filter()
+            .shuffle(
+                buffer_size=buffer_size1,
+            ): weight_genai
+            for d in domains
+            if DOMAIN_LABELS[d] != "laion"
+        }
+        dp = SampleMultiplexer({laion_dp: 1, **genai_dp}).shuffle(
+            buffer_size=buffer_size2,
+        )
+
+    if split == "train":
+        dp = dp.map(dp_to_tuple_train)
+    else:
+        dp = dp.map(dp_to_tuple_eval)
+
+    return dp
+
+
+def load_dataloader(
+    domains: list[int],
+    split: str,
+    batch_size: int = 32,
+    num_workers: int = 4,
+):
+    dp = load_dataset(domains, split)
+    # if split == "train":
+    #     dp = UnderSamplerIterDataPipe(dp, {0: 0.5, 1: 0.5}, seed=42)
+    dp = dp.batch(batch_size).collate()
+    dl = DataLoader(
+        dp,
+        batch_size=None,
+        num_workers=num_workers,
+        pin_memory=True,
+    )
+
+    return dl
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    args = parser.parse_args()
+
+    # testing code
+    dl = load_dataloader([0, 1], "train", num_workers=8)
+    y_dist = collections.Counter()
+    d_dist = collections.Counter()
+
+    for i, (img, y, d) in tqdm(enumerate(dl)):
+        if i % 100 == 0:
+            print(y, d)
+        if i == 400:
+            break
+        y_dist.update(y.numpy())
+        d_dist.update(d.numpy())
+
+    print("class label")
+    for label in sorted(y_dist):
+        frequency = y_dist[label] / sum(y_dist.values())
+        print(f"• {label}: {frequency:.2%} ({y_dist[label]})")
+
+    print("domain label")
+    for label in sorted(d_dist):
+        frequency = d_dist[label] / sum(d_dist.values())
+        print(f"• {label}: {frequency:.2%} ({d_dist[label]})")

src/images/diffusion_model_classifier.py

@@ -0,0 +1,293 @@
+import argparse
+import logging
+import os
+
+import pandas as pd
+import pytorch_lightning as pl
+import timm
+import torch
+import torch.nn.functional as F
+import torchvision.transforms as transforms
+from PIL import Image
+from pytorch_lightning.callbacks import (
+    EarlyStopping,
+    ModelCheckpoint,
+)
+from sklearn.metrics import roc_auc_score
+from torchmetrics import (
+    Accuracy,
+    Recall,
+)
+
+from .diffusion_data_loader import load_dataloader
+
+
+class ImageClassifier(pl.LightningModule):
+    def __init__(self, lmd=0):
+        super().__init__()
+        self.model = timm.create_model(
+            "resnet50",
+            pretrained=True,
+            num_classes=1,
+        )
+        self.accuracy = Accuracy(task="binary", threshold=0.5)
+        self.recall = Recall(task="binary", threshold=0.5)
+        self.validation_outputs = []
+        self.lmd = lmd
+
+    def forward(self, x):
+        return self.model(x)
+
+    def training_step(self, batch):
+        images, labels, _ = batch
+        outputs = self.forward(images).squeeze()
+
+        print(f"Shape of outputs (training): {outputs.shape}")
+        print(f"Shape of labels (training): {labels.shape}")
+
+        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
+        logging.info(f"Training Step - ERM loss: {loss.item()}")
+        loss += self.lmd * (outputs**2).mean()  # SD loss penalty
+        logging.info(f"Training Step - SD loss: {loss.item()}")
+        return loss
+
+    def validation_step(self, batch):
+        images, labels, _ = batch
+        outputs = self.forward(images).squeeze()
+
+        if outputs.shape == torch.Size([]):
+            return
+
+        print(f"Shape of outputs (validation): {outputs.shape}")
+        print(f"Shape of labels (validation): {labels.shape}")
+
+        loss = F.binary_cross_entropy_with_logits(outputs, labels.float())
+        preds = torch.sigmoid(outputs)
+        self.log("val_loss", loss, prog_bar=True, sync_dist=True)
+        self.log(
+            "val_acc",
+            self.accuracy(preds, labels.int()),
+            prog_bar=True,
+            sync_dist=True,
+        )
+        self.log(
+            "val_recall",
+            self.recall(preds, labels.int()),
+            prog_bar=True,
+            sync_dist=True,
+        )
+        output = {"val_loss": loss, "preds": preds, "labels": labels}
+        self.validation_outputs.append(output)
+        logging.info(f"Validation Step - Batch loss: {loss.item()}")
+        return output
+
+    def predict_step(self, batch):
+        images, label, domain = batch
+        outputs = self.forward(images).squeeze()
+        preds = torch.sigmoid(outputs)
+        return preds, label, domain
+
+    def on_validation_epoch_end(self):
+        if not self.validation_outputs:
+            logging.warning("No outputs in validation step to process")
+            return
+        preds = torch.cat([x["preds"] for x in self.validation_outputs])
+        labels = torch.cat([x["labels"] for x in self.validation_outputs])
+        if labels.unique().size(0) == 1:
+            logging.warning("Only one class in validation step")
+            return
+        auc_score = roc_auc_score(labels.cpu(), preds.cpu())
+        self.log("val_auc", auc_score, prog_bar=True, sync_dist=True)
+        logging.info(f"Validation Epoch End - AUC score: {auc_score}")
+        self.validation_outputs = []
+
+    def configure_optimizers(self):
+        optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0005)
+        return optimizer
+
+
+def load_image(image_path, transform=None):
+    image = Image.open(image_path).convert("RGB")
+
+    if transform:
+        image = transform(image)
+
+    return image
+
+
+def predict_single_image(image, model):
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    model.to(device)
+
+    image = image.to(device)
+
+    model.eval()
+
+    with torch.no_grad():
+        image = image.unsqueeze(0)
+        output = model(image).squeeze()
+        prediction = torch.sigmoid(output).item()
+
+    return prediction
+
+
+if __name__ == "__main__":
+    checkpoint_callback = ModelCheckpoint(
+        monitor="val_loss",
+        dirpath="./model_checkpoints/",
+        filename="image-classifier-{step}-{val_loss:.2f}",
+        save_top_k=3,
+        mode="min",
+        every_n_train_steps=1001,
+        enable_version_counter=True,
+    )
+
+    early_stop_callback = EarlyStopping(
+        monitor="val_loss",
+        patience=4,
+        mode="min",
+    )
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--ckpt_path",
+        help="checkpoint to continue from",
+        required=False,
+    )
+    parser.add_argument(
+        "--predict",
+        help="predict on test set",
+        action="store_true",
+    )
+    parser.add_argument("--reset", help="reset training", action="store_true")
+    parser.add_argument(
+        "--predict_image",
+        help="predict the class of a single image",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--image_path",
+        help="path to the image to predict",
+        type=str,
+        required=False,
+    )
+    parser.add_argument(
+        "--dir",
+        help="path to the images to predict",
+        type=str,
+        required=False,
+    )
+    parser.add_argument(
+        "--output_file",
+        help="path to output file",
+        type=str,
+        required=False,
+    )
+    args = parser.parse_args()
+
+    train_domains = [0, 1, 4]
+    val_domains = [0, 1, 4]
+    lmd_value = 0
+
+    if args.predict:
+        test_dl = load_dataloader(
+            [0, 1, 2, 3, 4],
+            "test",
+            batch_size=10,
+            num_workers=1,
+        )
+        model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
+        trainer = pl.Trainer()
+        predictions = trainer.predict(model, dataloaders=test_dl)
+        preds, labels, domains = zip(*predictions)
+        preds = torch.cat(preds).cpu().numpy()
+        labels = torch.cat(labels).cpu().numpy()
+        domains = torch.cat(domains).cpu().numpy()
+        print(preds.shape, labels.shape, domains.shape)
+        df = pd.DataFrame(
+            {"preds": preds, "labels": labels, "domains": domains},
+        )
+        filename = "preds-" + args.ckpt_path.split("/")[-1]
+        df.to_csv(f"outputs/{filename}.csv", index=False)
+    elif args.predict_image:
+        image_path = args.image_path
+        model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
+
+        # Define the transformations for the image
+        transform = transforms.Compose(
+            [
+                transforms.CenterCrop((256, 256)),
+                transforms.ToTensor(),
+            ],
+        )
+        image = load_image(image_path, transform)
+        prediction = predict_single_image(image, model)
+        print("prediction", prediction)
+
+        # Output the prediction
+        print(
+            f"Prediction for {image_path}: "
+            f"{'Human' if prediction <= 0.05 else 'Generated'}",
+        )
+    elif args.dir is not None:
+        predictions = []
+        model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
+        transform = transforms.Compose(
+            [
+                transforms.CenterCrop((256, 256)),
+                transforms.ToTensor(),
+            ],
+        )
+        for root, dirs, files in os.walk(os.path.abspath(args.dir)):
+            for f_name in files:
+                f = os.path.join(root, f_name)
+                print(f"Predicting: {f}")
+                p = predict_single_image(f, model)
+                predictions.append([f, f.split("/")[-2], p, p > 0.5])
+                print(f"--predicted: {p}")
+
+        df = pd.DataFrame(
+            predictions,
+            columns=["path", "folder", "pred", "class"],
+        )
+        df.to_csv(args.output_file, index=False)
+    else:
+        logging.basicConfig(
+            filename="training.log",
+            filemode="w",
+            level=logging.INFO,
+            force=True,
+        )
+        train_dl = load_dataloader(
+            train_domains,
+            "train",
+            batch_size=128,
+            num_workers=4,
+        )
+        logging.info("Training dataloader loaded")
+        val_dl = load_dataloader(
+            val_domains,
+            "val",
+            batch_size=128,
+            num_workers=4,
+        )
+        logging.info("Validation dataloader loaded")
+
+        if args.reset:
+            model = ImageClassifier.load_from_checkpoint(args.ckpt_path)
+        else:
+            model = ImageClassifier(lmd=lmd_value)
+        trainer = pl.Trainer(
+            callbacks=[checkpoint_callback, early_stop_callback],
+            max_steps=20000,
+            val_check_interval=1000,
+            check_val_every_n_epoch=None,
+        )
+        trainer.fit(
+            model=model,
+            train_dataloaders=train_dl,
+            val_dataloaders=val_dl,
+            ckpt_path=args.ckpt_path if not args.reset else None,
+        )

src/images/diffusion_utils_sampling.py

@@ -0,0 +1,94 @@
+import collections
+import random
+from typing import Callable
+
+from torchdata.datapipes.iter import IterDataPipe
+
+
+def get_second_entry(sample):
+    return sample[1]
+
+
+class UnderSamplerIterDataPipe(IterDataPipe):
+    """Dataset wrapper for under-sampling.
+
+    Copied from: https://github.com/MaxHalford/pytorch-resample/blob/master/pytorch_resample/under.py # noqa
+    Modified to work with multiple labels.
+
+    MIT License
+
+    Copyright (c) 2020 Max Halford
+
+    This method is based on rejection sampling.
+
+    Parameters:
+        dataset
+        desired_dist: The desired class distribution.
+            The keys are the classes whilst the
+            values are the desired class percentages.
+            The values are normalised so that sum up
+            to 1.
+        label_getter: A function that takes a sample and returns its label.
+        seed: Random seed for reproducibility.
+
+    Attributes:
+        actual_dist: The counts of the observed sample labels.
+        rng: A random number generator instance.
+
+    References:
+        - https://www.wikiwand.com/en/Rejection_sampling
+
+    """
+
+    def __init__(
+        self,
+        dataset: IterDataPipe,
+        desired_dist: dict,
+        label_getter: Callable = get_second_entry,
+        seed: int = None,
+    ):
+
+        self.dataset = dataset
+        self.desired_dist = {
+            c: p / sum(desired_dist.values()) for c, p in desired_dist.items()
+        }
+        self.label_getter = label_getter
+        self.seed = seed
+
+        self.actual_dist = collections.Counter()
+        self.rng = random.Random(seed)
+        self._pivot = None
+
+    def __iter__(self):
+
+        for dp in self.dataset:
+            y = self.label_getter(dp)
+
+            self.actual_dist[y] += 1
+
+            # To ease notation
+            f = self.desired_dist
+            g = self.actual_dist
+
+            # Check if the pivot needs to be changed
+            if y != self._pivot:
+                self._pivot = max(g.keys(), key=lambda y: f[y] / g[y])
+            else:
+                yield dp
+                continue
+
+            # Determine the sampling ratio if the observed label
+            # is not the pivot
+            M = f[self._pivot] / g[self._pivot]
+            ratio = f[y] / (M * g[y])
+
+            if ratio < 1 and self.rng.random() < ratio:
+                yield dp
+
+    @classmethod
+    def expected_size(cls, n, desired_dist, actual_dist):
+        M = max(
+            desired_dist.get(k) / actual_dist.get(k)
+            for k in set(desired_dist) | set(actual_dist)
+        )
+        return int(n / M)

src/images/image_demo.py

@@ -0,0 +1,73 @@
+import gradio as gr
+import torchvision.transforms as transforms
+from CNN_model_classifier import predict_cnn
+from diffusion_model_classifier import (
+    ImageClassifier,
+    predict_single_image,
+)
+
+gr.set_static_paths(paths=["samples/"])
+diffusion_model = (
+    "Diffusion/model_checkpoints/image-classifier-step=7007-val_loss=0.09.ckpt"
+)
+cnn_model = "CNN/model_checkpoints/blur_jpg_prob0.5.pth"
+
+
+def get_prediction_diffusion(image):
+    model = ImageClassifier.load_from_checkpoint(diffusion_model)
+
+    prediction = predict_single_image(image, model)
+    print(prediction)
+    return (prediction >= 0.001, prediction)
+
+
+def get_prediction_cnn(image):
+    prediction = predict_cnn(image, cnn_model)
+    return (prediction >= 0.5, prediction)
+
+
+def predict(inp):
+    # Define the transformations for the image
+    transform = transforms.Compose(
+        [
+            transforms.Resize((224, 224)),  # Image size expected by ResNet50
+            transforms.ToTensor(),
+            transforms.Normalize(
+                mean=[0.485, 0.456, 0.406],
+                std=[0.229, 0.224, 0.225],
+            ),
+        ],
+    )
+    image_tensor = transform(inp)
+    pred_diff, prob_diff = get_prediction_diffusion(image_tensor)
+    pred_cnn, prob_cnn = get_prediction_cnn(image_tensor)
+    verdict = (
+        "AI Generated" if (pred_diff or pred_cnn) else "No GenAI detected"
+    )
+    return (
+        f"<h1>{verdict}</h1>"
+        f"<ul>"
+        f"<li>Diffusion detection score: {prob_diff:.2} "
+        f"{'(MATCH)' if pred_diff else ''}</li>"
+        f"<li>CNN detection score: {prob_cnn:.1%} "
+        f"{'(MATCH)' if pred_cnn else ''}</li>"
+        f"</ul>"
+    )
+
+
+demo = gr.Interface(
+    title="AI-generated image detection",
+    description="Demo by NICT & Tokyo Techies ",
+    fn=predict,
+    inputs=gr.Image(type="pil"),
+    outputs=gr.HTML(),
+    examples=[
+        ["samples/fake_dalle.jpg", "Generated (Dall-E)"],
+        ["samples/fake_midjourney.png", "Generated (MidJourney)"],
+        ["samples/fake_stable.jpg", "Generated (Stable Diffusion)"],
+        ["samples/fake_cnn.png", "Generated (GAN)"],
+        ["samples/real.png", "Organic"],
+    ],
+)
+
+demo.launch()

src/main.py

@@ -0,0 +1,51 @@
+from texts.models import TextDetector
+
+
+def extract_text_and_images(path: str):
+    text_content = ""
+    image_paths = ""
+    return text_content, image_paths
+
+
+def process_document(document_path) -> list:
+    """
+    Processes a given document, separating text and images,
+    and then analyzes them.
+
+    Args:
+        document_path: Path to the document.
+
+    Returns:
+        A list containing the AI content percentage for text and images.
+    """
+
+    # Extract text and images from the document
+    text_content, image_paths = extract_text_and_images(document_path)
+
+    # Analyze text content
+    text_detector = TextDetector()
+    text_ai_content_percentage = text_detector.analyze_text(text_content)
+
+    # Analyze image content
+    image_ai_content_percentages = []
+    for image_path in image_paths:
+        # TODO: add image_detector class
+        # image_ai_content = image_detector.analyze_image(image_path)
+        image_ai_content = 100
+        image_ai_content_percentages.append(image_ai_content)
+
+    return [text_ai_content_percentage, image_ai_content_percentages]
+
+
+def main():
+    document_path = "../data.pdf"  # Replace with your document path
+    text_ai_content_percentage, image_ai_content_percentages = (
+        process_document(document_path)
+    )
+
+    print("Text AI Content Percentage:", text_ai_content_percentage)
+    print("Combined AI Content Percentage:", image_ai_content_percentages)
+
+
+if __name__ == "__main__":
+    main()

src/texts/MAGE/.gradio/flagged/dataset1.csv

@@ -0,0 +1,2 @@
+input text,AI-text detection,timestamp
+Does Chicago have any stores and does Joe live here?,"[{""token"": ""Does Chicago have any stores and does Joe live here?"", ""class_or_confidence"": ""human-written""}]",2024-12-09 13:40:10.255451

src/texts/MAGE/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

src/texts/MAGE/README.md

@@ -0,0 +1,258 @@
+<div align="center">
+<p align="center">
+  <img src="./figures/intro.png" width="75%" height="75%" />
+</p>
+</div>
+
+<div align="center">
+<h1><img src="./figures/title.png" width="30px" height="30px" style="display:inline;margin-right:10px;">MAGE: Machine-generated Text Detection in the Wild</h1>
+</div>
+
+<div align="center">
+<img src="https://img.shields.io/badge/Version-1.0.0-blue.svg" alt="Version"> 
+<img src="https://img.shields.io/badge/License-CC%20BY%204.0-green.svg" alt="License">
+<img src="https://img.shields.io/github/stars/yafuly/DeepfakeTextDetect?color=yellow" alt="Stars">
+<img src="https://img.shields.io/github/issues/yafuly/DeepfakeTextDetect?color=red" alt="Issues">
+
+<!-- **Authors:** -->
+<br>
+
+_**Yafu Li<sup>†</sup><sup>‡</sup>, Qintong Li<sup>§</sup>, Leyang Cui<sup>¶</sup>, Wei Bi<sup>¶</sup>,Zhilin Wang<sup>$</sup><br>**_
+
+_**Longyue Wang<sup>¶</sup>, Linyi Yang<sup>‡</sup>, Shuming Shi<sup>¶</sup>, Yue Zhang<sup>‡</sup><br>**_
+
+<!-- **Affiliations:** -->
+
+_<sup>†</sup> Zhejiang University,
+<sup>‡</sup> Westlake University,
+<sup>§</sup> The University of Hong Kong,
+<sup>$</sup> Jilin University,
+<sup>¶</sup> Tencent AI Lab_
+
+Presenting a comprehensive benchmark dataset designed to assess the proficiency of AI-generation detectors amidst real-world scenarios.
+Welcome to try detection via our **[online demo](https://detect.westlake.edu.cn)**!
+
+</div>
+
+## 📌 Table of Contents
+
+- [Introduction](#-introduction)
+- [Activities](#-activities)
+- [Dataset](#-dataset)
+- [Try Detection](#computer--try-detection)
+- [Data Samples](#-data-samples)
+- [Citation](#-citation)
+<!-- - [Contributing](#-contributing) -->
+
+## 🚀 Introduction
+
+Recent advances in large language models have enabled them to reach a level of text generation comparable to that of humans.
+These models show powerful capabilities across a wide range of content, including news article writing, story generation, and scientific writing.
+Such capability further narrows the gap between human-authored and machine-generated texts, highlighting the importance of machine-generated text detection to avoid potential risks such as fake news propagation and plagiarism.
+In practical scenarios, the detector faces texts from various domains or LLMs without knowing their sources.
+
+To this end, we build **a comprehensive testbed for deepfake text detection**, by gathering texts from various human writings and deepfake texts generated by different LLMs.
+This repository contains the data to testify deepfake detection methods described in our paper, [MAGE: Machine-generated Text Detection in the Wild](https://aclanthology.org/2024.acl-long.3/).
+Welcome to test your detection methods on our testbed!
+
+## 📅 Activities
+
+- 🎉 **May 16, 2024**: Our paper was accepted by ACL 2024!
+- 🎉 **June 19, 2023**: Update two 'wilder' testbeds! We go one step wilder by constructing an additional testset with texts from unseen domains generated by an unseen model, to testify the detection ability in more practical scenarios.
+  We consider four new datasets: CNN/DailyMail, DialogSum, PubMedQA and IMDb to test the detection of deepfake news, deepfake dialogues, deepfake scientific answers and deepfake movie reviews.
+  We sample 200 instances from each dataset and use a newly developed LLM, i.e., GPT-4, with specially designed prompts to create deepfake texts, establishing an "Unseen Domains & Unseen Model" scenario.
+  Previous work demonstrates that detection methods are vulnerable to being deceived by target texts.
+  Therefore, we also paraphrase each sentence individually for both human-written and machine-generated texts, forming an even more challenging testbed.
+  We adopt gpt-3.5-trubo as the zero-shot paraphraser and consider all paraphrased texts as machine-generated.
+- May 25, 2023: Initial dataset release including texts from 10 domains and 27 LLMs, contributing to 6 testbeds with increasing detection difficulty.
+
+## 📝 Dataset
+
+The dataset consists of **447,674** human-written and machine-generated texts from a wide range of sources in the wild:
+
+- Human-written texts from **10 datasets** covering a wide range of writing tasks, e.g., news article writing, story generation, scientific writing, etc.
+- Machine-generated texts generated by **27 mainstream LLMs** from 7 sources, e.g., OpenAI, LLaMA, and EleutherAI, etc.
+- **8 systematic testbed**s with increasing wildness and detection difficulty.
+
+### 📥 How to Get the Data
+
+#### 1. Huggingface
+
+You can access the full dataset, which includes the Cross-domains & Cross-models testbed and two additional wilder test sets, through the [Huggingface API](https://huggingface.co/datasets/yaful/MAGE):
+
+```python
+from datasets import load_dataset
+dataset = load_dataset("yaful/MAGE")
+```
+
+which includes traditional splits (train.csv, valid.csv and test.csv) and two wilder test sets (test_ood_set_gpt.csv and test_ood_set_gpt_para.csv).
+The csv files have three columns: text, label (0 for machine-generated and
+1 for human-written) and text source information (e.g., ''cmv_human'' denotes the text is written by humans,
+whereas ''roct_machine_continuation_flan_t5_large'' denotes the text is generated by ''flan_t5_large'' using continuation prompt).
+
+To obtain the 6 testbeds mentioned in our paper, simply apply the provided script:
+
+```shell
+python3 deployment/prepare_testbeds.py DATA_PATH
+```
+
+Replace ''DATA_PATH'' with the output data directory where you want to save the 6 testbeds.
+
+#### 2. Cloud Drive
+
+Alternatively, you can access the 6 testbeds by downloading them directly through [Google Drive](https://drive.google.com/drive/folders/1p09vDiEvoA-ZPmpqkB2WApcwMQWiiMRl?usp=sharing)
+or [Tencent Weiyun](https://share.weiyun.com/JUWQxF4H)：
+
+The folder contains 4 packages:
+
+- testbeds_processed.zip: 6 testbeds based on the ''processed'' version, which can be directly used for detecting in-distribution and out-of-distribution detection performance.
+- wilder_testsets.zip: 2 wilder test sets with texts processed, aiming for (1) detecting deepfake text generated by GPT-4, and (2) detecting deepfake text in paraphrased versions.
+- source.zip: Source texts of human-written texts and corresponding texts generated by LLMs, without filtering.
+- processed.zip: This is a refined version of the "source" that filters out low-quality texts and specifies sources as CSV file names. For example, the "cmv_machine_specified_gpt-3.5-trubo.csv" file contains texts from the CMV domain generated by the "gpt-3.5-trubo" model using specific prompts, while "cmv_human" includes human-written CMV texts.
+
+## :computer: Try Detection
+
+### Python Environment
+
+For deploying the Longformer detector or training your own detector using our data, simply install the following packages:
+
+```shell
+pip install transformers
+pip install datasets
+pip install clean-text # for data preprocessing
+```
+
+Or you can run:
+
+```shell
+pip install -r requirements.txt
+```
+
+### Model Access
+
+Our Longformer detector, which has been trained on the entire dataset, is now accessible through [Huggingface](https://huggingface.co/yaful/MAGE). Additionally, you can try detection directly using our [online demo](https://detect.westlake.edu.cn/).
+
+###
+
+We have refined the decision boundary based on out-of-distribution settings. To ensure optimal performance, we recommend preprocessing texts before sending them to the detector.
+
+```python
+import torch
+import os
+from transformers import AutoModelForSequenceClassification,AutoTokenizer
+from deployment import preprocess, detect
+
+# init
+device = 'cpu' # use 'cuda:0' if GPU is available
+# model_dir = "nealcly/detection-longformer" # model in our paper
+model_dir = "yaful/MAGE" # model in the online demo
+tokenizer = AutoTokenizer.from_pretrained(model_dir)
+model = AutoModelForSequenceClassification.from_pretrained(model_dir).to(device)
+
+text = "Apple's new credit card will begin a preview roll out today and will become available to all iPhone owners in the US later this month. A random selection of people will be allowed to go through the application process, which involves entering personal details which are sent to Goldman Sachs and TransUnion. Applications are approved or declined in less than a minute. The Apple Card is meant to be broadly accessible to every iPhone user, so the approval requirements will not be as strict as other credit cards. Once the application has been approved, users will be able to use the card immediately from the Apple Wallet app. The physical titanium card can be requested during setup for free, and it can be activated with NFC once it arrives."
+# preprocess
+text = preprocess(text)
+# detection
+result = detect(text,tokenizer,model,device)
+```
+
+### Detection Performance
+
+#### In-distribution Detection
+
+| Testbed                              | HumanRec | MachineRec | AvgRec | AUROC |
+| ------------------------------------ | -------- | ---------- | ------ | ----- |
+| White-box                            | 97.30%   | 95.91%     | 96.60% | 0.99  |
+| Arbitrary-domains & Model–specific   | 95.25%   | 96.94%     | 96.60% | 0.99  |
+| Fixed-domain & Arbitrary-models      | 89.78%   | 97.24%     | 93.51% | 0.99  |
+| Arbitrary-domains & Arbitrary-models | 82.80%   | 98.27%     | 90.53% | 0.99  |
+
+#### Out-of-distribution Detection
+
+| Testbed           | HumanRec | MachineRec | AvgRec | AUROC |
+| ----------------- | -------- | ---------- | ------ | ----- |
+| Unseen Model Sets | 83.31%   | 89.90%     | 86.61% | 0.95  |
+| Unseen Domains    | 38.05%   | 98.75%     | 68.40% | 0.93  |
+
+#### Wilder Testsets
+
+| Testbed                       | HumanRec | MachineRec | AvgRec | AUROC |
+| ----------------------------- | -------- | ---------- | ------ | ----- |
+| Unseen Domains & Unseen Model | 88.78%   | 84.12%     | 86.54% | 0.94  |
+| Paraphrase                    | 88.78%   | 37.05%     | 62.92% | 0.75  |
+
+## 📄 Data Samples
+
+All instances are stored as rows in a csv format, with each row consiting of 3 columns: _Text_, _Label_ (0 for machine-generated and 1 for human-written) and _Index_ (indexes in the original data source, used restore alignment after filtering).
+
+#### News Article
+
+| Text                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             | Label |
+| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
+| Apple's new credit card will begin a preview roll out today and will become available to all iPhone owners in the US later this month. A random selection of people will be allowed to go through the application process, which involves entering personal details which are sent to Goldman Sachs and TransUnion. Applications are approved or declined in less than a minute. The Apple Card is meant to be broadly accessible to every iPhone user, so the approval requirements will not be as strict as other credit cards. Once the application has been approved, users will be able to use the card immediately from the Apple Wallet app. The physical titanium card can be requested during setup for free, and it can be activated with NFC once it arrives.                                                                                                                                                                                                                                                                                                                                                                                         | 1     |
+| Apple's new credit card will begin a preview roll out today and will become available to all iPhone owners in the US later this month. A random selection of people who applied for an Apple Card beta account last week will be sent invitations just before April 26, when everyone else can start using it too. The Apple Card is designed with privacy in mind: users aren't able to link their cards or view detailed transaction histories online as some other services do. The app itself also stores no personal data on your phone, only a virtual version in its secure payment system so you don't have to give Apple access to any sensitive details. It uses machine learning-based algorithms that learn from how you spend to provide personalized recommendations about your spending habits. Your transactions are stored offline on your device, the company says, while information like email addresses remains encrypted during transit between your devices and the cloud. And if you ever lose your physical card, Apple has said there's a way to temporarily disable the card without affecting payments until the actual one arrives. | 0     |
+| Today marks the beginning of a new era of financial technology: Apple Card is now available to all users in the United States. The long-awaited credit card from Apple, which was announced earlier this year, is now available for everyone to sign up and use. With features such as cashback on purchases and robust security measures, Apple Card could revolutionize how people make payments. This could be the start of a new wave of digital payment options.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            | 0     |
+
+#### Opinion Statement
+
+| Text                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          | Label |
+| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
+| Look, I know this is a touchy subject, and while people might say I'm young and didn't understand the complexity of wars, just hear me out. Vietnam was essentially a communist state, due to influences from China and USSR, which were alliances (the former is debatable) of Vietnam during the war. After the war, our country has suffered multiple economic depressions, and famines due to the incompetence of our liberally named Communist Party. Granted the South Vietnam government wasn't any better, but what the U.S wanted for Vietnam was for the best. I understand that, technically the US did not wage war with our people, but stood against the spread of communism in Asia, and with our strategic location, a battle surely followed. The US did not deliberately invaded our country. And look at what they did to the world. Defeated the Nazis and fascist countries, uplifted South Korea, Japan (which were both smaller and less resourceful than my country) to their respectable position on the world map today. And what had the sole communist party in my country done? Nothing but left our people in the struggle of a third-world country. And China is still brazenly harassing our borders and seas to this very day, just because our army is incapable of standing up for themselves. Please tell me if I was wrong and why the North won was a good idea. Edit: My view has changed. It was not simple as I thought it was. Generally it can be summarized into those points: involvement of China, motives and war crimes committed by the US, and there was no hope in the governing system. Communism has not helped our people a bit, but no one can really advocates for America either. We as a nation should look to develop our own path. Insights are still very much appreciated. And thanks for the discussions.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      | 1     |
+| Look, I know this is a touchy subject, and while people might say I'm young and didn't understand the complexity of wars, just hear me out. Vietnam was essentially a lost war. A war where we fought against the communists, but lost, after years of fighting and thousands of lives lost. We were a technologically advanced nation, but outmatched by the communists who were determined to destroy us. And they almost did. So when I think about Iraq, I can't help but compare it to Vietnam. And the only thing I'm seeing is our forces being put in a situation where they can't win. Let's start with the weapons. I'm not a weapons expert by any means, so I don't know all the fine details. But the simple facts are this: the communists had the Russians, and we had the U.S. (and other allies). Well, the communists have just as many weapons as we have, if not more. I understand that we can win by outnumbering them, but that is very difficult. Most likely we will have to use sophisticated weapons, but then we get into the tricky area of international law. Can you really justify dropping a bomb on a country that has a pretty advanced military force (think of North Korea, for example)? The answer might be yes, because if you don't do that you're handing the war to them, but then you have to ask yourself if you really want to start that slippery slope. Now there are some people who think that if we just let the terrorists have their way with us, then we will send a message to the world. Well, if that's the case, then what about the message we send by having weapons that are supposedly sophisticated enough to kill entire countries? You can't send a message by allowing innocent people to die, and if you want to kill innocent people, then you might as well start killing people at home. So there are people who say we should use these weapons in Iraq, and there are others who say we shouldn't, and there are the people who have their own ideas. But the one thing I know is this: we are in a very difficult position. We don't have the technology to back up our claims that we are the good guys, and we don't want to lose by being outmatched, so the only thing we can do is back out of the war. But this brings up a very interesting point. I wonder if Bush, who has been preaching against the communists, is going to back out of Iraq. And if he doesn't, what kind of message does that send? I know that he wants to send a message to the rest of the world, but do we really want to send that message? If we do, then what about the message we send by supporting one of the richest nations in the world, and supporting war that many of us don't even want? I know that many of you disagree with me, and I'm sorry if this is rude, but I'm just trying to get people to think. I'm not trying to be mean, and I know that I'm not right, but at least I have something to say. I know that I can't change anything, but I know that I can at least try. | 0     |
+| It is understandable that you may wish the United States had won the Vietnam War, however, it is important to recognize that the Vietnam War was a complex conflict with many political and social implications. In reality, it is impossible to predict what would have happened if the U.S. had won the war. The war could have potentially resulted in more loss of life and suffering for the Vietnamese people. It is also important to consider that the war united the Vietnamese people and eventually led to the reunification of Vietnam in 1976, which could not have occurred if the U.S. had been victorious. Therefore, while it can be tempting to look back on history and wish for a different outcome, it is important to recognize the complexities of the Vietnam War and the positive outcomes that have come from it.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | 0     |
+
+#### Long-form Answer
+
+| Text                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          | Label |
+| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
+| That is called bootstrap problem. How can you program something when no software exists that lets you program things. And how can a computer read what to do, if it doesn't know how to read. The answer is that you have to write a really simple program yourself, onto the hardware. It never changes for a computer, and is used every time you turn it on. That tiny program doesn't do anything except tell every part of the computer what it is and where it can get the stuff it needs. This includes really basic stuff, like storage adresses and and how to read them. From then on, the hardware can look up how to use the screen, how to read the keyboard, all those things. It's of course a bit more complicated than that, but once you have that first spark going, you can build up on that and program away.,We did use keyboards. They just weren't connected to the computer. You typed in your command on what was basically a typewriter which then"punched" the information onto cards. These were known as Hollerith Punch Cards - the machine looked like this: URL0 You then took the stack of cards very carefully to the computer hopper and fed them in. They had to stay in the same order they were punched for your program to work.', "Originally, computers were little more than some cathodes connected by cables. Programming them was done by changing the connections. A little later, computers had readers that took in cards with holes in certain distances, serving as binary input. I imagine.the first keyboard simply used that reader's interface,Well, for a pretty user-friendly way, there's the original [Altair 8800] (URL1). Note the switches on the front; when entering a program, you would set them to a position representing a number representing an operation of the computer, then press a button to store it in the memory and proceed to the next position. When you were done you'd press some other button to start the program you just "typed." Of course, all would be lost when you turned off the power.', "Via use of physical digital circuitry. I kid you not. As a project for an introductory course to digitial systems we had to make a ping pong game explicitly from digital circuits using specific IC's (and the basic NOTXORORAND gate IC's only). There was no code whatsoever in this system. Once you manage that, everything else is easy.,Punched cards preceded computers by [decades] (URL2). They could be used with sorters and counters to answer fairly sophisticated questions, as long as the data fit on the card. When business computers become useful they had the cards and the keyboards used to fill them out ready and waiting. | 1     |
+| The original programmers probably used a different way to make the computer do what they wanted it to do because keyboards did not exist yet. They might have used a different type of machine to give commands, or they might have used punch cards or other methods to write down instructions that the computer could understand. The keyboard was invented later to make it easier and faster for people to tell a computer what to do.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | 0     |
+| That is called bootstrap problem. How can you program something when no software exists that lets you program things. And how can a computer read what to do, if it has no instructions to begin with? One possible solution to the bootstrap problem is to start with a set of very basic instructions written in machine language (the language that computers speak), and use those instructions to build more complex instructions and programs. This process is also known as bootstrapping or self-hosting. For example, a programmer could start with the most basic machine instructions to add, subtract, and compare numbers, and then use those instructions to create more complex operations like conditional statements, loops, and functions. Over time, more complex programs and programming languages can be built upon this foundation. Another solution is to use a pre-existing programming language or tool to create the initial instructions or software needed to bootstrap a new system. This approach is common in the development of operating systems, where code is initially written in a higher-level language like C, and then compiled into machine code that can be run on a computer. Overall, there are several ways to approach the bootstrap problem, and the solution will typically depend on the specific goals and constraints of the project.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     | 0     |
+
+#### Story Generation
+
+| Text                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       | Label |
+| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
+| Thomas loves science fiction, and is pleased to find himself sitting by the park entrance with Arthur C. Clarke's " Fountains of Paradise " open in his lap. He must have jogged there, he thinks to himself as he admires his brand new black-and-white Nikes. He stretches out in his black joggers and turns the page. " But there was no substitute for reality, one should beware of imitations ," he reads before shutting the book. Thomas ponders what he has read as he looks to the right; not a single car can be seen. The street appears infinite in length and the buildings fade in to the distance with it. He stands and begins his first step down the street. His movement halts when he hears a young voice behind him, " You look thirsty mister. Would you like some lemonade? " Thomas walks back past the park entrance and over to the lemonade stand, wondering how he had not noticed it before. It is beautiful, the entrance; but the park is closed now. Thomas stares up at the gates in awe. Thomas is interrupted again by the child, " 5.50, please. " Thomas looks at the counter, flustered. " I'll have the punch instead. " As the child pours the purple drink in to the cup, Thomas reaches in his pocket finding a five dollar bill and three quarters. " Keep the change ," Thomas says as he picks up his drink. Thomas sips and the sky slowly dims. He feels his breath drawn away from him as a comet sails over the park entrance. And Heaven's Gate opens. | 1     |
+| Thomas loves science fiction, and is pleased to find himself sitting by the park entrance with Arthur C. Clarke's " Fountains of Paradise " open in his lap. He must have been reading for quite a while, as it's getting dark, and the other night-time park visitors are beginning to emerge. He gets up to leave, and on his way out finds a very tiny boy walking around in circles, trying to find his parents. The little boy is quite distressed, and Thomas takes him to the park office, which is locked. Thomas finally remembers that he's got a cell phone in his pocket, and calls the number on the sign. The woman on the other end is very kind, and promises to come help the boy right away. Thomas is pleased to have been able to help, and heads off to the train station to go home. On the train, his eyes are tired, and he falls asleep. At the end of the chapter, we find out that the woman on the phone was the boy's grandmother. The boy was seven years old, and his parents had taken him to the park for a picnic. The boy had started walking around in circles when he couldn't find his mother and father again.                                                                                                                                                                                                                                                                                                                                                      | 0     |
+| Jeff was a normal guy, living a normal life. He had a family, a job, and a few friends. But above all else, he wasn't religious. He rarely thought about religion, and when he did, it was with a kind of apathy. One day, Jeff died unexpectedly. He woke up in an unfamiliar place, surrounded by people he didn't know. He was confused, but no one seemed to mind. As he looked around, Jeff noticed that everyone was dressed differently and speaking different languages. Then it hit him - he had died and gone to the afterlife. But something else struck him: none of these people were from his own religion. In fact, he didn't recognize any of the religions here. Then it dawned on him - this wasn't the afterlife of his religion, it was the afterlife of the religion whose tenets he had followed most closely, knowingly or not. He had lived his life without being religious, but had unknowingly followed a certain set of beliefs. Now, in the afterlife, he was among those who had done the same. Jeff found himself feeling strangely comforted in this new place. He realized that even though his faith had been different than others', its core values were still very much the same. This newfound understanding filled Jeff with peace and joy, and he felt like he had really come home.                                                                                                                                                                               | 0     |
+
+#### Scientific Writing
+
+| Text                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              | Label |
+| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
+| Although deep-learning-based methods have markedly improved the performance of speech separation over the past few years, it remains an open question how to integrate multi-channel signals for speech separation. We propose two methods, namely, early-fusion and late-fusion methods, to integrate multi-channel information based on the time-domain audio separation network, which has been proven effective in single-channel speech separation. We also propose channel-sequential-transfer learning, which is a transfer learning framework that applies the parameters trained for a lower-channel network as the initial values of a higher-channel network. For fair comparison, we evaluated our proposed methods using a spatialized version of the wsj0-2mix dataset, which is open-sourced. It was found that our proposed methods can outperform multi-channel deep clustering and improve the performance proportionally to the number of microphones. It was also proven that the performance of the late-fusion method is consistently higher than that of the single-channel method regardless of the angle difference between speakers.                                                                                    | 1     |
+| Although deep learning has achieved appealing results on several machine learning tasks, most of the models are deterministic at inference, limiting their application to single-modal settings. We propose a novel probabilistic deep learning model, namely Probabilistic Interpretation Network (PIN), which enables multi-modal inference, uncertainty quantification, and sample-based exploration by extracting latent representations from multiple modalities (e.g. vision and language) and modeling their dependencies via a probabilistic graphical model. PIN is a flexible framework that can be used to train interpretable multi-modal models as well as handle modalities in an unsupervised setting. We apply PIN to a wide variety of tasks including out-of-distribution detection, visual question answering and goal-driven dialogue. We present a new evaluation metric for goal-driven dialogue and show that PIN is capable of handling both modalities and uncertainty in this setting.                                                                                                                                                                                                                                  | 0     |
+| Although deep learning has achieved appealing results on several machine learning tasks, most of the models are deterministic at inference, limiting their application to single-modal settings. We propose a novel approach that allows to perform probabilistic inference with deep learning models. Our method is based on a variational autoencoder (VAE) and uses a mixture of Gaussians as a prior distribution for the latent variable. The VAE is trained by maximising a variational lower bound on the data log-likelihood, which can be seen as an evidence lower bound (ELBO). We introduce a novel approach to learn this ELBO, which is based on the re-parameterisation trick. This trick allows us to use standard gradient descent techniques to optimise the ELBO and consequently obtain a probabilistic latent representation for the data. We evaluate our model on a variety of datasets, including images, text, and speech. Our results show that our approach achieves comparable performance to existing deterministic models, while providing a probabilistic interpretation of the input data. Moreover, we demonstrate that our approach yields better generalisation ability when compared to deterministic models. | 0     |
+
+## 📚 Citation
+
+If you use this dataset in your research, please cite it as follows:
+
+```bibtex
+@inproceedings{li-etal-2024-mage,
+    title = "{MAGE}: Machine-generated Text Detection in the Wild",
+    author = "Li, Yafu  and
+      Li, Qintong  and
+      Cui, Leyang  and
+      Bi, Wei  and
+      Wang, Zhilin  and
+      Wang, Longyue  and
+      Yang, Linyi  and
+      Shi, Shuming  and
+      Zhang, Yue",
+    editor = "Ku, Lun-Wei  and
+      Martins, Andre  and
+      Srikumar, Vivek",
+    booktitle = "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
+    month = aug,
+    year = "2024",
+    address = "Bangkok, Thailand",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2024.acl-long.3",
+    doi = "10.18653/v1/2024.acl-long.3",
+    pages = "36--53",
+}
+```
+
+We welcome contributions to improve this dataset! If you have any questions or feedback, please feel free to reach out at [email protected].

src/texts/MAGE/app.py

@@ -0,0 +1,74 @@
+from transformers import pipeline
+from difflib import Differ
+from transformers import AutoModelForSequenceClassification,AutoTokenizer
+from deployment import preprocess, detect
+import gradio as gr
+
+ner_pipeline = pipeline("ner")
+
+
+def ner(text):
+    output = ner_pipeline(text)
+    output = [
+        {'entity': 'I-LOC', 'score': 0.9995369, 'index': 2, 'word': 'Chicago', 'start': 5, 'end': 12},
+        {'entity': 'I-PER', 'score': 0.99527764, 'index': 8, 'word': 'Joe', 'start': 38, 'end': 41}
+        ]
+    print(output)
+    return {"text": text, "entities": output}
+
+def diff_texts(text1, text2):
+    d = Differ()
+    return [
+        (token[2:], token[0] if token[0] != " " else None)
+        for token in d.compare(text1, text2)
+    ]
+    
+out = diff_texts(
+    "The quick brown fox jumped over the lazy dogs.",
+    "The fast brown fox jumps over lazy dogs.")
+print(out)
+
+
+def separate_characters_with_mask(text, mask):
+  """Separates characters in a string and pairs them with a mask sign.
+
+  Args:
+    text: The input string.
+
+  Returns:
+    A list of tuples, where each tuple contains a character and a mask.
+  """
+
+  return [(char, mask) for char in text]
+
+
+def detect_ai_text(text):
+    text = preprocess(text)
+    result = detect(text,tokenizer,model,device)
+    print(result)
+    output = separate_characters_with_mask(text, result)
+    return output
+
+# init
+device = 'cpu' # use 'cuda:0' if GPU is available
+# model_dir = "nealcly/detection-longformer" # model in our paper
+model_dir = "yaful/MAGE" # model in the online demo
+tokenizer = AutoTokenizer.from_pretrained(model_dir)
+model = AutoModelForSequenceClassification.from_pretrained(model_dir).to(device)
+examples = ["Apple's new credit card will begin a preview roll out today and will become available to all iPhone owners in the US later this month. A random selection of people will be allowed to go through the application process, which involves entering personal details which are sent to Goldman Sachs and TransUnion. Applications are approved or declined in less than a minute. The Apple Card is meant to be broadly accessible to every iPhone user, so the approval requirements will not be as strict as other credit cards. Once the application has been approved, users will be able to use the card immediately from the Apple Wallet app. The physical titanium card can be requested during setup for free, and it can be activated with NFC once it arrives."]
+
+demo = gr.Interface(detect_ai_text,
+             gr.Textbox(
+                 label="input text",
+                 placeholder="Enter text here...",
+                 lines=5,
+                 ),
+             gr.HighlightedText(
+                 label="AI-text detection",
+                 combine_adjacent=True,
+                 show_legend=True,
+                 color_map={"machine-generated": "red", "human-written": "green"}
+                 ),
+             examples=examples)
+
+demo.launch(share=True)

src/texts/MAGE/deployment/__init__.py

@@ -0,0 +1 @@
+from .utils import *

src/texts/MAGE/deployment/prepare_testbeds.py

@@ -0,0 +1,348 @@
+import csv
+import os
+import sys
+from collections import defaultdict
+import random
+from datasets import load_dataset
+
+set_names = [
+    "cmv",
+    "yelp",
+    "xsum",
+    "tldr",
+    "eli5",
+    "wp",
+    "roct",
+    "hswag",
+    "squad",
+    "sci_gen",
+]
+
+oai_list = [
+    # openai
+    "gpt-3.5-trubo",
+    "text-davinci-003",
+    "text-davinci-002",
+]
+llama_list = ["_7B", "_13B", "_30B", "_65B"]
+glm_list = [
+    "GLM130B",
+]
+flan_list = [
+    # flan_t5,
+    "flan_t5_small",
+    "flan_t5_base",
+    "flan_t5_large",
+    "flan_t5_xl",
+    "flan_t5_xxl",
+]
+
+opt_list = [
+    # opt,
+    "opt_125m",
+    "opt_350m",
+    "opt_1.3b",
+    "opt_2.7b",
+    "opt_6.7b",
+    "opt_13b",
+    "opt_30b",
+    "opt_iml_30b",
+    "opt_iml_max_1.3b",
+]
+bigscience_list = [
+    "bloom_7b",
+    "t0_3b",
+    "t0_11b",
+]
+eleuther_list = [
+    "gpt_j",
+    "gpt_neox",
+]
+model_sets = [
+    oai_list,
+    llama_list,
+    glm_list,
+    flan_list,
+    opt_list,
+    bigscience_list,
+    eleuther_list,
+]
+
+data_dir = sys.argv[1]
+dataset = load_dataset("yaful/DeepfakeTextDetect")
+if not os.path.exists(data_dir):
+    os.makedirs(data_dir)
+"""
+csv_path = f"{data_dir}/train.csv"
+train_results = list(csv.reader(open(csv_path,encoding='utf-8-sig')))[1:]
+csv_path = f"{data_dir}/valid.csv"
+valid_results = list(csv.reader(open(csv_path,encoding='utf-8-sig')))[1:]
+csv_path = f"{data_dir}/test.csv"
+test_results = list(csv.reader(open(csv_path,encoding='utf-8-sig')))[1:]
+"""
+train_results = [
+    (row["text"], str(row["label"]), row["src"]) for row in list(dataset["train"])
+]
+valid_results = [
+    (row["text"], str(row["label"]), row["src"]) for row in list(dataset["validation"])
+]
+test_results = [
+    (row["text"], str(row["label"]), row["src"]) for row in list(dataset["test"])
+]
+merge_dict = {
+    "train": (train_results, 800),
+    "valid": (valid_results, 100),
+    "test": (test_results, 100),
+}
+
+
+test_ood_gpt = dataset["test_ood_gpt"]
+test_ood_gpt_para = dataset["test_ood_gpt_para"]
+test_ood_gpt.to_csv(os.path.join(data_dir, "test_ood_gpt.csv"))
+test_ood_gpt_para.to_csv(os.path.join(data_dir, "test_ood_gpt_para.csv"))
+
+
+# make domain-specific_model-specific (gpt_j)
+def prepare_domain_specific_model_specific():
+    tgt_model = "gpt_j"
+    testbed_dir = f"{data_dir}/domain_specific_model_specific"
+    sub_results = defaultdict(lambda: defaultdict(list))
+    print("# preparing domain-specific & model-specific ...")
+    for name in set_names:
+        print(f"## preparing {name} ...")
+        for split in ["train", "valid", "test"]:
+            split_results, split_count = merge_dict[split]
+            count = 0
+            for res in split_results:
+                info = res[2]
+                res = res[:2]
+                if name in info:
+                    # human-written
+                    if res[1] == "1" and count <= split_count:
+                        sub_results[name][split].append(res)
+                    # machine-generated
+                    if tgt_model in info:
+                        assert res[1] == "0"
+                        sub_results[name][split].append(res)
+                    count += 1
+
+        sub_dir = f"{testbed_dir}/{name}"
+        os.makedirs(sub_dir, exist_ok=True)
+        for split in ["train", "valid", "test"]:
+            print(f"{split} set: {len(sub_results[name][split])}")
+            rows = sub_results[name][split]
+            row_head = [["text", "label"]]
+            rows = row_head + rows
+            tmp_path = f"{sub_dir}/{split}.csv"
+            with open(tmp_path, "w", newline="", encoding="utf-8-sig") as f:
+                csvw = csv.writer(f)
+                csvw.writerows(rows)
+
+
+# make domain_specific_cross_models
+def prepare_domain_specific_cross_models():
+    testbed_dir = f"{data_dir}/domain_specific_cross_models"
+    sub_results = defaultdict(lambda: defaultdict(list))
+
+    print("# preparing domain_specific_cross_models ...")
+    for name in set_names:
+        print(f"## preparing {name} ...")
+        for split in ["train", "valid", "test"]:
+            split_results, split_count = merge_dict[split]
+            for res in split_results:
+                info = res[2]
+                res = res[:2]
+                if name in info:
+                    # human-written
+                    if res[1] == "1":
+                        sub_results[name][split].append(res)
+                    # machine-generated
+                    else:
+                        sub_results[name][split].append(res)
+
+        sub_dir = f"{testbed_dir}/{name}"
+        os.makedirs(sub_dir, exist_ok=True)
+        for split in ["train", "valid", "test"]:
+            print(f"{split} set: {len(sub_results[name][split])}")
+            rows = sub_results[name][split]
+            row_head = [["text", "label"]]
+            rows = row_head + rows
+            tmp_path = f"{sub_dir}/{split}.csv"
+            with open(tmp_path, "w", newline="", encoding="utf-8-sig") as f:
+                csvw = csv.writer(f)
+                csvw.writerows(rows)
+
+
+# make cross_domains_model_specific
+def prepare_cross_domains_model_specific():
+    print("# preparing cross_domains_model_specific ...")
+    for model_patterns in model_sets:
+        sub_dir = f"{data_dir}/cross_domains_model_specific/model_{model_patterns[0]}"
+        os.makedirs(sub_dir, exist_ok=True)
+        # model_pattern = dict.fromkeys(model_pattern)
+        _tmp = " ".join(model_patterns)
+        print(f"## preparing {_tmp} ...")
+
+        ood_pos_test_samples = []
+        out_split_samples = defaultdict(list)
+        for split in ["train", "valid", "test"]:
+            rows = merge_dict[split][0]
+            # print(f"Original {split} set length: {len(rows)}")
+
+            out_rows = []
+            for row in rows:
+                valid = False
+                srcinfo = row[2]
+                if row[1] == "1":  # appending all positive samples
+                    valid = True
+                for pattern in model_patterns:
+                    if pattern in srcinfo:
+                        valid = True
+                        break
+                if valid:
+                    out_rows.append(row)
+                    # out_rows.append(row+[srcinfo[0]])
+
+            out_split_samples[split] = out_rows
+
+        for split in ["train", "valid", "test"]:
+            random.seed(1)
+            rows = out_split_samples[split]
+            pos_rows = [r for r in rows if r[1] == "1"]
+            neg_rows = [r for r in rows if r[1] == "0"]
+            len_neg = len(neg_rows)
+            random.shuffle(pos_rows)
+            out_split_samples[split] = pos_rows[:len_neg] + neg_rows
+
+        for split in ["train", "valid", "test"]:
+            out_rows = [e[:-1] for e in out_split_samples[split]]
+            print(f"{split} set: {len(out_rows)} ...")
+            # xxx
+            tgt_path = f"{sub_dir}/{split}.csv"
+            with open(tgt_path, "w", newline="", encoding="utf-8-sig") as f:
+                csvw = csv.writer(f)
+                csvw.writerows([["text", "label"]] + out_rows)
+
+
+# make cross_domains_cross_models
+def prepare_cross_domains_cross_models():
+    print("# preparing cross_domains_cross_models ...")
+    testbed_dir = f"{data_dir}/cross_domains_cross_models"
+    os.makedirs(testbed_dir, exist_ok=True)
+    for split in ["train", "valid", "test"]:
+        csv_path = f"{testbed_dir}/{split}.csv"
+
+        with open(csv_path, "w", newline="", encoding="utf-8-sig") as f:
+            rows = [row[:-1] for row in merge_dict[split][0]]
+            print(f"{split} set: {len(rows)} ...")
+            csvw = csv.writer(f)
+            csvw.writerows([["text", "label"]] + rows)
+
+
+# make unseen_models
+def prepare_unseen_models():
+    print("# preparing unseen_models ...")
+    for model_patterns in model_sets:
+        sub_dir = f"{data_dir}/unseen_models/unseen_model_{model_patterns[0]}"
+        os.makedirs(sub_dir, exist_ok=True)
+        _tmp = " ".join(model_patterns)
+        print(f"## preparing ood-models {_tmp} ...")
+
+        ood_pos_test_samples = []
+        out_split_samples = defaultdict(list)
+        for split in ["train", "valid", "test", "test_ood"]:
+            data_name = split if split != "test_ood" else "test"
+            rows = merge_dict[data_name][0]
+
+            out_rows = []
+            for row in rows:
+                valid = False
+                srcinfo = row[2]
+                for pattern in model_patterns:
+                    if split != "test_ood":
+                        if pattern in srcinfo:
+                            valid = False
+                            break
+                        valid = True
+                    else:
+                        if pattern in srcinfo:
+                            valid = True
+                            break
+                if valid:
+                    out_rows.append(row)
+
+            out_split_samples[split] = out_rows
+
+        random.seed(1)
+        test_rows = out_split_samples["test"]
+        test_pos_rows = [r for r in test_rows if r[1] == "1"]
+        test_neg_rows = [r for r in test_rows if r[1] == "0"]
+        len_aug = len(out_split_samples["test_ood"])
+        # print(len_aug)
+        random.shuffle(test_pos_rows)
+        # out_split_samples['test'] = test_pos_rows[len_aug:] + test_neg_rows
+        out_split_samples["test_ood"] = (
+            test_pos_rows[:len_aug] + out_split_samples["test_ood"]
+        )
+
+        for split in ["train", "valid", "test", "test_ood"]:
+            out_rows = [e[:-1] for e in out_split_samples[split]]
+            print(f"{split} set: {len(out_rows)}")
+
+            tgt_path = f"{sub_dir}/{split}.csv"
+            with open(tgt_path, "w", newline="", encoding="utf-8-sig") as f:
+                csvw = csv.writer(f)
+                csvw.writerows([["text", "label"]] + out_rows)
+
+
+# make unseen_domains
+def prepare_unseen_domains():
+    print("# preparing unseen_domains ...")
+
+    testbed_dir = f"{data_dir}/unseen_domains"
+    sub_results = defaultdict(lambda: defaultdict(list))
+
+    for name in set_names:
+        sub_dir = f"{data_dir}/unseen_domains/unseen_domain_{name}"
+        os.makedirs(sub_dir, exist_ok=True)
+
+        print(f"## preparing ood-domains {name} ...")
+
+        ood_pos_test_samples = []
+        out_split_samples = defaultdict(list)
+        for split in ["train", "valid", "test", "test_ood"]:
+            data_name = split if split != "test_ood" else "test"
+            rows = merge_dict[data_name][0]
+
+            out_rows = []
+            for row in rows:
+                srcinfo = row[2]
+                valid = True if name in srcinfo else False
+                valid = not valid if split != "test_ood" else valid
+                if valid:
+                    out_rows.append(row)
+
+            out_split_samples[split] = out_rows
+
+        for split in ["train", "valid", "test", "test_ood"]:
+            out_rows = [e[:-1] for e in out_split_samples[split]]
+            print(f"{split} set: {len(out_rows)}")
+            tgt_path = f"{sub_dir}/{split}.csv"
+            with open(tgt_path, "w", newline="", encoding="utf-8-sig") as f:
+                csvw = csv.writer(f)
+                csvw.writerows([["text", "label"]] + out_rows)
+
+
+# prepare 6 testbeds
+prepare_domain_specific_model_specific()
+print("-" * 100)
+prepare_domain_specific_cross_models()
+print("-" * 100)
+prepare_cross_domains_model_specific()
+print("-" * 100)
+prepare_cross_domains_cross_models()
+print("-" * 100)
+prepare_unseen_models()
+print("-" * 100)
+prepare_unseen_domains()
+print("-" * 100)

src/texts/MAGE/deployment/utils.py

@@ -0,0 +1,294 @@
+import re
+import torch
+from cleantext import clean
+from itertools import chain
+
+class MosesPunctNormalizer:
+    """
+    This is a Python port of the Moses punctuation normalizer from
+    https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/normalize-punctuation.perl
+    """
+
+    EXTRA_WHITESPACE = [  # lines 21 - 30
+        (r"\r", r""),
+        (r"\(", r" ("),
+        (r"\)", r") "),
+        (r" +", r" "),
+        (r"\) ([.!:?;,])", r")\g<1>"),
+        (r"\( ", r"("),
+        (r" \)", r")"),
+        (r"(\d) %", r"\g<1>%"),
+        (r" :", r":"),
+        (r" ;", r";"),
+    ]
+
+    NORMALIZE_UNICODE_IF_NOT_PENN = [(r"`", r"'"), (r"''", r' " ')]  # lines 33 - 34
+
+    NORMALIZE_UNICODE = [  # lines 37 - 50
+        ("„", r'"'),
+        ("“", r'"'),
+        ("”", r'"'),
+        ("–", r"-"),
+        ("—", r" - "),
+        (r" +", r" "),
+        ("´", r"'"),
+        ("([a-zA-Z])‘([a-zA-Z])", r"\g<1>'\g<2>"),
+        ("([a-zA-Z])’([a-zA-Z])", r"\g<1>'\g<2>"),
+        ("‘", r"'"),
+        ("‚", r"'"),
+        ("’", r"'"),
+        (r"''", r'"'),
+        ("´´", r'"'),
+        ("…", r"..."),
+    ]
+
+    FRENCH_QUOTES = [  # lines 52 - 57
+        ("\u00A0«\u00A0", r'"'),
+        ("«\u00A0", r'"'),
+        ("«", r'"'),
+        ("\u00A0»\u00A0", r'"'),
+        ("\u00A0»", r'"'),
+        ("»", r'"'),
+    ]
+
+    HANDLE_PSEUDO_SPACES = [  # lines 59 - 67
+        ("\u00A0%", r"%"),
+        ("nº\u00A0", "nº "),
+        ("\u00A0:", r":"),
+        ("\u00A0ºC", " ºC"),
+        ("\u00A0cm", r" cm"),
+        ("\u00A0\\?", "?"),
+        ("\u00A0\\!", "!"),
+        ("\u00A0;", r";"),
+        (",\u00A0", r", "),
+        (r" +", r" "),
+    ]
+
+    EN_QUOTATION_FOLLOWED_BY_COMMA = [(r'"([,.]+)', r'\g<1>"')]
+
+    DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA = [
+        (r',"', r'",'),
+        (r'(\.+)"(\s*[^<])', r'"\g<1>\g<2>'),  # don't fix period at end of sentence
+    ]
+
+    DE_ES_CZ_CS_FR = [
+        ("(\\d)\u00A0(\\d)", r"\g<1>,\g<2>"),
+    ]
+
+    OTHER = [
+        ("(\\d)\u00A0(\\d)", r"\g<1>.\g<2>"),
+    ]
+
+    # Regex substitutions from replace-unicode-punctuation.perl
+    # https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl
+    REPLACE_UNICODE_PUNCTUATION = [
+        ("，", ","),
+        (r"。\s*", ". "),
+        ("、", ","),
+        ("”", '"'),
+        ("“", '"'),
+        ("∶", ":"),
+        ("：", ":"),
+        ("？", "?"),
+        ("《", '"'),
+        ("》", '"'),
+        ("）", ")"),
+        ("！", "!"),
+        ("（", "("),
+        ("；", ";"),
+        ("」", '"'),
+        ("「", '"'),
+        ("０", "0"),
+        ("１", "1"),
+        ("２", "2"),
+        ("３", "3"),
+        ("４", "4"),
+        ("５", "5"),
+        ("６", "6"),
+        ("７", "7"),
+        ("８", "8"),
+        ("９", "9"),
+        (r"．\s*", ". "),
+        ("～", "~"),
+        ("’", "'"),
+        ("…", "..."),
+        ("━", "-"),
+        ("〈", "<"),
+        ("〉", ">"),
+        ("【", "["),
+        ("】", "]"),
+        ("％", "%"),
+    ]
+
+    def __init__(
+        self,
+        lang="en",
+        penn=True,
+        norm_quote_commas=True,
+        norm_numbers=True,
+        pre_replace_unicode_punct=False,
+        post_remove_control_chars=False,
+    ):
+        """
+        :param language: The two-letter language code.
+        :type lang: str
+        :param penn: Normalize Penn Treebank style quotations.
+        :type penn: bool
+        :param norm_quote_commas: Normalize quotations and commas
+        :type norm_quote_commas: bool
+        :param norm_numbers: Normalize numbers
+        :type norm_numbers: bool
+        """
+        self.substitutions = [
+            self.EXTRA_WHITESPACE,
+            self.NORMALIZE_UNICODE,
+            self.FRENCH_QUOTES,
+            self.HANDLE_PSEUDO_SPACES,
+        ]
+
+        if penn:  # Adds the penn substitutions after extra_whitespace regexes.
+            self.substitutions.insert(1, self.NORMALIZE_UNICODE_IF_NOT_PENN)
+
+        if norm_quote_commas:
+            if lang == "en":
+                self.substitutions.append(self.EN_QUOTATION_FOLLOWED_BY_COMMA)
+            elif lang in ["de", "es", "fr"]:
+                self.substitutions.append(self.DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA)
+
+        if norm_numbers:
+            if lang in ["de", "es", "cz", "cs", "fr"]:
+                self.substitutions.append(self.DE_ES_CZ_CS_FR)
+            else:
+                self.substitutions.append(self.OTHER)
+
+        self.substitutions = list(chain(*self.substitutions))
+
+        self.pre_replace_unicode_punct = pre_replace_unicode_punct
+        self.post_remove_control_chars = post_remove_control_chars
+
+    def normalize(self, text):
+        """
+        Returns a string with normalized punctuation.
+        """
+        # Optionally, replace unicode puncts BEFORE normalization.
+        if self.pre_replace_unicode_punct:
+            text = self.replace_unicode_punct(text)
+
+        # Actual normalization.
+        for regexp, substitution in self.substitutions:
+            # print(regexp, substitution)
+            text = re.sub(regexp, substitution, str(text))
+            # print(text)
+
+        # Optionally, replace unicode puncts BEFORE normalization.
+        if self.post_remove_control_chars:
+            text = self.remove_control_chars(text)
+
+        return text.strip()
+
+    def replace_unicode_punct(self, text):
+        for regexp, substitution in self.REPLACE_UNICODE_PUNCTUATION:
+            text = re.sub(regexp, substitution, str(text))
+        return text
+
+    def remove_control_chars(self, text):
+        return regex.sub(r"\p{C}", "", text)
+
+def _tokenization_norm(text):
+    text = text.replace(
+        ' ,', ',').replace(
+        ' .', '.').replace(
+        ' ?', '?').replace(
+        ' !', '!').replace(
+        ' ;', ';').replace(
+        ' \'', '\'').replace(
+        ' ’ ', '\'').replace(
+        ' :', ':').replace(
+        '<newline>', '\n').replace(
+        '`` ', '"').replace(
+        ' \'\'', '"').replace(
+        '\'\'', '"').replace(
+        '.. ', '... ').replace(
+        ' )', ')').replace(
+        '( ', '(').replace(
+        ' n\'t', 'n\'t').replace(
+        ' i ', ' I ').replace(
+        ' i\'', ' I\'').replace(
+        '\\\'', '\'').replace(
+        '\n ', '\n').strip()
+    return text
+
+
+def _clean_text(text):
+    # remove PLM special tokens
+    plm_special_tokens = r'(\<pad\>)|(\<s\>)|(\<\/s\>)|(\<unk\>)|(\<\|endoftext\|\>)'
+    text = re.sub(plm_special_tokens, "", text)
+
+    # normalize puncuations
+    moses_norm = MosesPunctNormalizer()
+    text = moses_norm.normalize(text)
+    
+    # normalize tokenization
+    text = _tokenization_norm(text)
+    
+    # remove specific text patterns, e.g,, url, email and phone number
+    text = clean(text,
+        fix_unicode=True,               # fix various unicode errors
+        to_ascii=True,                  # transliterate to closest ASCII representation
+        lower=False,                     # lowercase text
+        no_line_breaks=True,           # fully strip line breaks as opposed to only normalizing them
+        no_urls=True,                  # replace all URLs with a special token
+        no_emails=True,                # replace all email addresses with a special token
+        no_phone_numbers=True,         # replace all phone numbers with a special token
+        no_numbers=False,               # replace all numbers with a special token
+        no_digits=False,                # replace all digits with a special token
+        no_currency_symbols=False,      # replace all currency symbols with a special token
+        no_punct=False,                 # remove punctuations
+        replace_with_punct="",          # instead of removing punctuations you may replace them
+        replace_with_url="",
+        replace_with_email="",
+        replace_with_phone_number="",
+        replace_with_number="<NUMBER>",
+        replace_with_digit="<DIGIT>",
+        replace_with_currency_symbol="<CUR>",
+        lang="en"                       # set to 'de' for German special handling
+    )
+    
+    # keep common puncts only
+    punct_pattern = r'[^ A-Za-z0-9.?!,:;\-\[\]\{\}\(\)\'\"]'
+    text = re.sub(punct_pattern, '', text)
+    # remove specific patterns
+    spe_pattern = r'[-\[\]\{\}\(\)\'\"]{2,}'
+    text = re.sub(spe_pattern, '', text)
+    # remove redundate spaces
+    text = " ".join(text.split())
+    return text
+
+def _rm_line_break(text):
+    text = text.replace("\n","\\n")
+    text = re.sub(r'(?:\\n)*\\n', r'\\n', text)
+    text = re.sub(r'^.{0,3}\\n', '', text)
+    text = text.replace("\\n"," ")
+    return text
+
+def preprocess(text):
+    text = _rm_line_break(text)
+    text = _clean_text(text)
+    return text
+
+
+def detect(input_text,tokenizer,model,device='cuda:0',th=-3.08583984375):
+    label2decisions = {
+        0: "machine-generated",
+        1: "human-written",
+    }
+    tokenize_input = tokenizer(input_text)
+    tensor_input = torch.tensor([tokenize_input["input_ids"]]).to(device)
+    outputs = model(tensor_input)
+    is_machine = -outputs.logits[0][0].item()
+    if is_machine < th:
+        decision = 0
+    else:
+        decision = 1
+
+    return label2decisions[decision]

src/texts/MAGE/main.py

@@ -0,0 +1,65 @@
+from transformers import AutoModelForSequenceClassification,AutoTokenizer
+import datasets
+from deployment import preprocess, detect
+import csv
+import pandas as pd
+
+# init
+device = 'cpu' # use 'cuda:0' if GPU is available
+# model_dir = "nealcly/detection-longformer" # model in our paper
+model_dir = "yaful/MAGE" # model in the online demo
+tokenizer = AutoTokenizer.from_pretrained(model_dir)
+model = AutoModelForSequenceClassification.from_pretrained(model_dir).to(device)
+
+# text = "Apple's new credit card will begin a preview roll out today and will become available to all iPhone owners in the US later this month. A random selection of people will be allowed to go through the application process, which involves entering personal details which are sent to Goldman Sachs and TransUnion. Applications are approved or declined in less than a minute. The Apple Card is meant to be broadly accessible to every iPhone user, so the approval requirements will not be as strict as other credit cards. Once the application has been approved, users will be able to use the card immediately from the Apple Wallet app. The physical titanium card can be requested during setup for free, and it can be activated with NFC once it arrives."
+# # preprocess
+# text = preprocess(text)
+# # detection
+# result = detect(text,tokenizer,model,device)
+# print(result)
+
+# ds = datasets.load_dataset('RealTimeData/bbc_news_alltime', '2020-02')
+# test 100 samples from (RealTimeData/bbc_news_alltime', '2020-02') 
+# df = pd.read_csv('query_result.csv')
+# content_column = df['content']
+# count = 0
+
+# for content in content_column:
+#     # preprocess
+#     text = preprocess(content)
+#     # detection
+#     result = detect(text, tokenizer, model, device)
+#     if result == "human-written":
+#         count +=1
+    
+#     print(count)
+# print(count)
+
+
+# ds = datasets.load_dataset('yaful/MAGE', 'test')
+# ds.save_to_disk("MAGE_data")
+# splits = list(ds.keys())
+# print(splits)
+
+ds = datasets.load_from_disk("MAGE_data")
+
+#filtered_data = ds['test'].filter(lambda x: x['src'] == 'xsum_human')
+
+human_data = [example['text'] for example in ds['test'] if example['src'] == 'xsum_human']
+human_data = human_data[0:100]
+
+machine_data = [example['text'] for example in ds['test'] if example['src'] == 'xsum_machine_topical_gpt-3.5-trubo']
+machine_data = machine_data[0:100]
+
+count = 0
+for content in machine_data:
+    # preprocess
+    text = preprocess(content)
+    # detection
+    result = detect(text, tokenizer, model, device)
+    print(result)
+    if result == "human-written":  # machine-generated
+        count +=1
+    
+    print(count)
+print(count)

src/texts/MAGE/requirements.txt

@@ -0,0 +1,51 @@
+accelerate==0.24.1
+aiohttp==3.9.1
+aiosignal==1.3.1
+async-timeout==4.0.3
+attrs==23.1.0
+certifi==2023.11.17
+charset-normalizer==3.3.2
+clean-text==0.6.0
+click==8.1.7
+datasets==2.15.0
+dill==0.3.7
+emoji==1.7.0
+filelock==3.13.1
+frozenlist==1.4.0
+fsspec==2023.10.0
+ftfy==6.1.3
+huggingface-hub==0.19.4
+idna==3.6
+joblib==1.3.2
+multidict==6.0.4
+multiprocess==0.70.15
+nltk==3.8.1
+numpy==1.26.2
+packaging==23.2
+pandas==2.1.3
+Pillow==10.1.0
+pip==23.3.1
+psutil==5.9.6
+pyarrow==14.0.1
+pyarrow-hotfix==0.6
+python-dateutil==2.8.2
+pytz==2023.3.post1
+PyYAML==6.0.1
+regex==2023.10.3
+requests==2.31.0
+safetensors==0.4.1
+setuptools==68.0.0
+six==1.16.0
+tokenizers==0.15.0
+#torch==1.13.1+cu116
+#torchaudio==0.13.1+cu116
+#torchvision==0.14.1+cu116
+tqdm==4.66.1
+transformers==4.35.2
+typing_extensions==4.8.0
+tzdata==2023.3
+urllib3==2.1.0
+wcwidth==0.2.12
+wheel==0.41.2
+xxhash==3.4.1
+yarl==1.9.3

src/texts/MAGE/training/longformer/main.py

@@ -0,0 +1,666 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright 2020 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Finetuning the library models for sequence classification on GLUE."""
+# You can also adapt this script on your own text classification task. Pointers for this are left as comments.
+
+import logging
+import os
+import random
+import sys
+from dataclasses import dataclass, field
+from typing import Optional
+
+import datasets
+import numpy as np
+from datasets import load_dataset, load_metric
+
+import transformers
+from transformers import (
+    AutoConfig,
+    AutoModelForSequenceClassification,
+    AutoTokenizer,
+    DataCollatorWithPadding,
+    EvalPrediction,
+    HfArgumentParser,
+    PretrainedConfig,
+    Trainer,
+    TrainingArguments,
+    default_data_collator,
+    set_seed,
+)
+from transformers.trainer_utils import get_last_checkpoint
+from transformers.utils import check_min_version
+from transformers.utils.versions import require_version
+
+
+os.environ['CURL_CA_BUNDLE'] = ''
+# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
+check_min_version("4.9.0")
+
+require_version("datasets>=1.8.0",
+                "To fix: pip install -r examples/pytorch/text-classification/requirements.txt")
+
+task_to_keys = {
+    "cola": ("sentence", None),
+    "mnli": ("premise", "hypothesis"),
+    "mrpc": ("sentence1", "sentence2"),
+    "qnli": ("question", "sentence"),
+    "qqp": ("question1", "question2"),
+    "rte": ("sentence1", "sentence2"),
+    "sst2": ("sentence", None),
+    "stsb": ("sentence1", "sentence2"),
+    "wnli": ("sentence1", "sentence2"),
+}
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class DataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+    Using `HfArgumentParser` we can turn this class
+    into argparse arguments to be able to specify them on
+    the command line.
+    """
+
+    task_name: Optional[str] = field(
+        default=None,
+        metadata={"help": "The name of the task to train on: " +
+                  ", ".join(task_to_keys.keys())},
+    )
+    dataset_name: Optional[str] = field(
+        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
+    )
+    dataset_config_name: Optional[str] = field(
+        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
+    )
+    max_seq_length: int = field(
+        default=128,
+        metadata={
+            "help": "The maximum total input sequence length after tokenization. Sequences longer "
+            "than this will be truncated, sequences shorter will be padded."
+        },
+    )
+    overwrite_cache: bool = field(
+        default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."}
+    )
+    pad_to_max_length: bool = field(
+        default=True,
+        metadata={
+            "help": "Whether to pad all samples to `max_seq_length`. "
+            "If False, will pad the samples dynamically when batching to the maximum length in the batch."
+        },
+    )
+    max_train_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": "For debugging purposes or quicker training, truncate the number of training examples to this "
+            "value if set."
+        },
+    )
+    max_eval_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
+            "value if set."
+        },
+    )
+    max_predict_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": "For debugging purposes or quicker training, truncate the number of prediction examples to this "
+            "value if set."
+        },
+    )
+    train_file: Optional[str] = field(
+        default=None, metadata={"help": "A csv or a json file containing the training data."}
+    )
+    validation_file: Optional[str] = field(
+        default=None, metadata={"help": "A csv or a json file containing the validation data."}
+    )
+    test_file: Optional[str] = field(default=None, metadata={
+                                     "help": "A csv or a json file containing the test data."})
+    from_scratch: bool = field(
+        default=False,
+        metadata={
+            "help": "set true to not load weights from pretrained models."
+        },
+    )
+    # do_eval: Optional[bool] = field(
+    #     default=False, metadata={"help": "do evaluation."}
+    # )
+
+    def __post_init__(self):
+        if self.task_name is not None:
+            self.task_name = self.task_name.lower()
+            if self.task_name not in task_to_keys.keys():
+                raise ValueError(
+                    "Unknown task, you should pick one in " + ",".join(task_to_keys.keys()))
+        elif self.dataset_name is not None:
+            pass
+        elif self.train_file is None or self.validation_file is None:
+            raise ValueError(
+                "Need either a GLUE task, a training/validation file or a dataset name.")
+        else:
+            train_extension = self.train_file.split(".")[-1]
+            assert train_extension in [
+                "csv", "json"], "`train_file` should be a csv or a json file."
+            validation_extension = self.validation_file.split(".")[-1]
+            assert (
+                validation_extension == train_extension
+            ), "`validation_file` should have the same extension (csv or json) as `train_file`."
+
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
+    """
+
+    model_name_or_path: str = field(
+        metadata={
+            "help": "Path to pretrained model or model identifier from huggingface.co/models"}
+    )
+    config_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
+    )
+    tokenizer_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
+    )
+    cache_dir: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+    )
+    use_fast_tokenizer: bool = field(
+        default=True,
+        metadata={
+            "help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
+    )
+    model_revision: str = field(
+        default="main",
+        metadata={
+            "help": "The specific model version to use (can be a branch name, tag name or commit id)."},
+    )
+    use_auth_token: bool = field(
+        default=False,
+        metadata={
+            "help": "Will use the token generated when running `transformers-cli login` (necessary to use this script "
+            "with private models)."
+        },
+    )
+
+
+def main():
+    # See all possible arguments in src/transformers/training_args.py
+    # or by passing the --help flag to this script.
+    # We now keep distinct sets of args, for a cleaner separation of concerns.
+
+    parser = HfArgumentParser(
+        (ModelArguments, DataTrainingArguments, TrainingArguments))
+    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        model_args, data_args, training_args = parser.parse_json_file(
+            json_file=os.path.abspath(sys.argv[1]))
+    else:
+        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+
+    if data_args.validation_file == data_args.test_file:
+        training_args.do_eval = False
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        handlers=[logging.StreamHandler(sys.stdout)],
+    )
+
+    log_level = training_args.get_process_log_level()
+    # training_args["report_to"] = None # disable integrations
+    logger.setLevel(log_level)
+    datasets.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.enable_default_handler()
+    transformers.utils.logging.enable_explicit_format()
+
+    # Log on each process the small summary:
+    logger.warning(
+        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
+        + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
+    )
+    logger.info(f"Training/evaluation parameters {training_args}")
+
+    # Detecting last checkpoint.
+    last_checkpoint = None
+    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
+        last_checkpoint = get_last_checkpoint(training_args.output_dir)
+        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
+            raise ValueError(
+                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
+                "Use --overwrite_output_dir to overcome."
+            )
+        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
+            logger.info(
+                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
+                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
+            )
+
+    # Set seed before initializing model.
+    set_seed(training_args.seed)
+
+    # Get the datasets: you can either provide your own CSV/JSON training and evaluation files (see below)
+    # or specify a GLUE benchmark task (the dataset will be downloaded automatically from the datasets Hub).
+    #
+    # For CSV/JSON files, this script will use as labels the column called 'label' and as pair of sentences the
+    # sentences in columns called 'sentence1' and 'sentence2' if such column exists or the first two columns not named
+    # label if at least two columns are provided.
+    #
+    # If the CSVs/JSONs contain only one non-label column, the script does single sentence classification on this
+    # single column. You can easily tweak this behavior (see below)
+    #
+    # In distributed training, the load_dataset function guarantee that only one local process can concurrently
+    # download the dataset.
+    if data_args.task_name is not None:
+        # Downloading and loading a dataset from the hub.
+        raw_datasets = load_dataset(
+            "glue", data_args.task_name, cache_dir=model_args.cache_dir)
+    elif data_args.dataset_name is not None:
+        # Downloading and loading a dataset from the hub.
+        raw_datasets = load_dataset(
+            data_args.dataset_name, data_args.dataset_config_name, cache_dir=model_args.cache_dir
+        )
+    else:
+        # Loading a dataset from your local files.
+        # CSV/JSON training and evaluation files are needed.
+        data_files = {"train": data_args.train_file,
+                      "validation": data_args.validation_file}
+
+        # Get the test dataset: you can provide your own CSV/JSON test file (see below)
+        # when you use `do_predict` without specifying a GLUE benchmark task.
+        if training_args.do_predict:
+            if data_args.test_file is not None:
+                train_extension = data_args.train_file.split(".")[-1]
+                test_extension = data_args.test_file.split(".")[-1]
+                assert (
+                    test_extension == train_extension
+                ), "`test_file` should have the same extension (csv or json) as `train_file`."
+                data_files["test"] = data_args.test_file
+
+            else:
+                raise ValueError(
+                    "Need either a GLUE task or a test file for `do_predict`.")
+
+        for key in data_files.keys():
+            logger.info(f"load a local file for {key}: {data_files[key]}")
+
+        if data_args.train_file.endswith(".csv"):
+            # Loading a dataset from local csv files
+            raw_datasets = load_dataset(
+                "csv", data_files=data_files, cache_dir=model_args.cache_dir)
+        else:
+            # Loading a dataset from local json files
+            raw_datasets = load_dataset(
+                "json", data_files=data_files, cache_dir=model_args.cache_dir)
+    # See more about loading any type of standard or custom dataset at
+    # https://huggingface.co/docs/datasets/loading_datasets.html.
+
+    # Labels
+    if data_args.task_name is not None:
+        is_regression = data_args.task_name == "stsb"
+        if not is_regression:
+            label_list = raw_datasets["train"].features["label"].names
+            num_labels = len(label_list)
+        else:
+            num_labels = 1
+    else:
+        # Trying to have good defaults here, don't hesitate to tweak to your needs.
+        is_regression = raw_datasets["train"].features["label"].dtype in [
+            "float32", "float64"]
+        if is_regression:
+            num_labels = 1
+        else:
+            # A useful fast method:
+            # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.unique
+            label_list = raw_datasets["train"].unique("label")
+            label_list.sort()  # Let's sort it for determinism
+            num_labels = len(label_list)
+
+    # Load pretrained model and tokenizer
+    #
+    # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently
+    # download model & vocab.
+    config = AutoConfig.from_pretrained(
+        model_args.config_name if model_args.config_name else model_args.model_name_or_path,
+        num_labels=num_labels,
+        finetuning_task=data_args.task_name,
+        cache_dir=model_args.cache_dir,
+        revision=model_args.model_revision,
+        use_auth_token=True if model_args.use_auth_token else None,
+    )
+    tokenizer = AutoTokenizer.from_pretrained(
+        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
+        cache_dir=model_args.cache_dir,
+        use_fast=model_args.use_fast_tokenizer,
+        revision=model_args.model_revision,
+        use_auth_token=True if model_args.use_auth_token else None,
+    )
+    if not data_args.from_scratch:
+        model = AutoModelForSequenceClassification.from_pretrained(
+            model_args.model_name_or_path,
+            from_tf=bool(".ckpt" in model_args.model_name_or_path),
+            config=config,
+            cache_dir=model_args.cache_dir,
+            revision=model_args.model_revision,
+            use_auth_token=True if model_args.use_auth_token else None,
+            ignore_mismatched_sizes=True,
+        )
+    else:
+        model = AutoModelForSequenceClassification.from_config(
+            config=config,
+            # ignore_mismatched_sizes=True,
+        )
+    # Preprocessing the raw_datasets
+    sentence1_key, sentence2_key = "text", None
+    # if data_args.task_name is not None:
+    #     sentence1_key, sentence2_key = task_to_keys[data_args.task_name]
+    # else:
+    #     # Again, we try to have some nice defaults but don't hesitate to tweak to your use case.
+    #     non_label_column_names = [name for name in raw_datasets["train"].column_names if name != "label"]
+    #     if "sentence1" in non_label_column_names and "sentence2" in non_label_column_names:
+    #         sentence1_key, sentence2_key = "sentence1", "sentence2"
+    #     else:
+    #         if len(non_label_column_names) >= 2:
+    #             sentence1_key, sentence2_key = non_label_column_names[:2]
+    #         else:
+    #             sentence1_key, sentence2_key = non_label_column_names[0], None
+
+    # Padding strategy
+    if data_args.pad_to_max_length:
+        padding = "max_length"
+    else:
+        # We will pad later, dynamically at batch creation, to the max sequence length in each batch
+        padding = False
+
+    # Some models have set the order of the labels to use, so let's make sure we do use it.
+    label_to_id = None
+    if (
+        model.config.label2id != PretrainedConfig(
+            num_labels=num_labels).label2id
+        and data_args.task_name is not None
+        and not is_regression
+    ):
+        # Some have all caps in their config, some don't.
+        label_name_to_id = {
+            k.lower(): v for k, v in model.config.label2id.items()}
+        if list(sorted(label_name_to_id.keys())) == list(sorted(label_list)):
+            label_to_id = {
+                i: int(label_name_to_id[label_list[i]]) for i in range(num_labels)}
+        else:
+            logger.warning(
+                "Your model seems to have been trained with labels, but they don't match the dataset: ",
+                f"model labels: {list(sorted(label_name_to_id.keys()))}, dataset labels: {list(sorted(label_list))}."
+                "\nIgnoring the model labels as a result.",
+            )
+    elif data_args.task_name is None and not is_regression:
+        label_to_id = {v: i for i, v in enumerate(label_list)}
+
+    if label_to_id is not None:
+        model.config.label2id = label_to_id
+        model.config.id2label = {
+            id: label for label, id in config.label2id.items()}
+
+    if data_args.max_seq_length > tokenizer.model_max_length:
+        logger.warning(
+            f"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the"
+            f"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}."
+        )
+    max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)
+
+    def preprocess_function(examples):
+        # Tokenize the texts
+        args = (
+            (examples[sentence1_key],) if sentence2_key is None else (
+                examples[sentence1_key], examples[sentence2_key])
+        )
+        result = tokenizer(*args, padding=padding,
+                           max_length=max_seq_length, truncation=True)
+        # print('finish')
+        # print(examples[sentence1_key])
+        # Map labels to IDs (not necessary for GLUE tasks)
+
+        result["label"] = examples['label']
+        return result
+
+    with training_args.main_process_first(desc="dataset map pre-processing"):
+        raw_datasets = raw_datasets.map(
+            preprocess_function,
+            batched=True,
+            load_from_cache_file=not data_args.overwrite_cache,
+            desc="Running tokenizer on dataset",
+        )
+    if training_args.do_train:
+        if "train" not in raw_datasets:
+            raise ValueError("--do_train requires a train dataset")
+        train_dataset = raw_datasets["train"]
+        if data_args.max_train_samples is not None:
+            train_dataset = train_dataset.select(
+                range(data_args.max_train_samples))
+    # print(training_args.do_eval)
+    # xxx
+    if training_args.do_eval:
+        if "validation" not in raw_datasets and "validation_matched" not in raw_datasets:
+            raise ValueError("--do_eval requires a validation dataset")
+        eval_dataset = raw_datasets["validation_matched" if data_args.task_name ==
+                                    "mnli" else "validation"]
+        if data_args.max_eval_samples is not None:
+            eval_dataset = eval_dataset.select(
+                range(data_args.max_eval_samples))
+
+    if training_args.do_predict or data_args.task_name is not None or data_args.test_file is not None:
+        if "test" not in raw_datasets and "test_matched" not in raw_datasets:
+            raise ValueError("--do_predict requires a test dataset")
+        predict_dataset = raw_datasets["test_matched" if data_args.task_name ==
+                                       "mnli" else "test"]
+        if data_args.max_predict_samples is not None:
+            predict_dataset = predict_dataset.select(
+                range(data_args.max_predict_samples))
+
+    # Log a few random samples from the training set:
+    if training_args.do_train:
+        for index in random.sample(range(len(train_dataset)), 3):
+            logger.info(
+                f"Sample {index} of the training set: {train_dataset[index]}.")
+
+    # Get the metric function
+    # if data_args.task_name is not None:
+    #     metric = load_metric("glue", data_args.task_name)
+    # else:
+    #     metric = load_metric("accuracy", cache_dir='./evaluate')
+
+    # You can define your custom compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a
+    # predictions and label_ids field) and has to return a dictionary string to float.
+    def compute_metrics(p: EvalPrediction):
+        preds = p.predictions[0] if isinstance(
+            p.predictions, tuple) else p.predictions
+        preds = np.squeeze(
+            preds) if is_regression else np.argmax(preds, axis=1)
+        # if data_args.task_name is not None:
+        #     result = metric.compute(predictions=preds, references=p.label_ids)
+        #     if len(result) > 1:
+        #         result["combined_score"] = np.mean(list(result.values())).item()
+        #     return result
+        if is_regression:
+            return {"mse": ((preds - p.label_ids) ** 2).mean().item()}
+        else:
+            accuracy = (preds == p.label_ids).astype(np.float32).mean().item()
+            TP = ((preds == p.label_ids) & (preds == 1)
+                  ).astype(np.float32).sum().item()
+            TN = ((preds == p.label_ids) & (preds == 0)
+                  ).astype(np.float32).sum().item()
+            FN = ((preds != p.label_ids) & (preds == 0)
+                  ).astype(np.float32).sum().item()
+            FP = ((preds != p.label_ids) & (preds == 1)
+                  ).astype(np.float32).sum().item()
+
+            # metric_precision = load_metric("precision", cache_dir='./evaluate')
+            # precision = metric_precision.compute(predictions=preds, references=p.label_ids, average='macro')
+            # metric_recall = load_metric("recall", cache_dir='./evaluate')
+            # recall = metric_recall.compute(predictions=preds, references=p.label_ids, average='macro')
+            # metric_fscore = load_metric("f1", cache_dir='./evaluate')
+            # f1score = metric_fscore.compute(predictions=preds, references=p.label_ids, average='macro')
+            # print("-"*100)
+            try:
+                precision = TP / (TP+FP)
+                recall = TP / (TP+FN)
+                f1score = 2*precision*recall/(precision+recall)
+                print(f'precision:{precision}/recall"{recall}/f1:{f1score}')
+                precision = TN / (TN+FN)
+                recall = TN / (TN+FP)
+                f1score = 2*precision*recall/(precision+recall)
+                print(f'precision:{precision}/recall"{recall}/f1:{f1score}')
+            except:
+                print("float division by zero ...")
+            # return {
+            #     "precision": precision,
+            #     "recall": recall,
+            #     "f1": f1score
+            #     }
+            return {
+                "accuracy": accuracy
+            }
+    # Data collator will default to DataCollatorWithPadding, so we change it if we already did the padding.
+    if data_args.pad_to_max_length:
+        data_collator = default_data_collator
+    elif training_args.fp16:
+        data_collator = DataCollatorWithPadding(
+            tokenizer, pad_to_multiple_of=8)
+    else:
+        data_collator = None
+
+    # Initialize our Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset if training_args.do_train else None,
+        eval_dataset=eval_dataset if training_args.do_eval else None,
+        compute_metrics=compute_metrics,
+        tokenizer=tokenizer,
+        data_collator=data_collator,
+    )
+
+    # Training
+    if training_args.do_train:
+        checkpoint = None
+        if training_args.resume_from_checkpoint is not None:
+            checkpoint = training_args.resume_from_checkpoint
+        elif last_checkpoint is not None:
+            checkpoint = last_checkpoint
+        train_result = trainer.train(resume_from_checkpoint=checkpoint)
+        metrics = train_result.metrics
+        max_train_samples = (
+            data_args.max_train_samples if data_args.max_train_samples is not None else len(
+                train_dataset)
+        )
+        metrics["train_samples"] = min(max_train_samples, len(train_dataset))
+
+        trainer.save_model()  # Saves the tokenizer too for easy upload
+
+        trainer.log_metrics("train", metrics)
+        trainer.save_metrics("train", metrics)
+        trainer.save_state()
+
+    # Evaluation
+    if training_args.do_eval:
+        logger.info("*** Evaluate ***")
+
+        # Loop to handle MNLI double evaluation (matched, mis-matched)
+        tasks = [data_args.task_name]
+        eval_datasets = [eval_dataset]
+        if data_args.task_name == "mnli":
+            tasks.append("mnli-mm")
+            eval_datasets.append(raw_datasets["validation_mismatched"])
+
+        for eval_dataset, task in zip(eval_datasets, tasks):
+            metrics = trainer.evaluate(eval_dataset=eval_dataset)
+
+            max_eval_samples = (
+                data_args.max_eval_samples if data_args.max_eval_samples is not None else len(
+                    eval_dataset)
+            )
+            metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
+
+            trainer.log_metrics("eval", metrics)
+            trainer.save_metrics("eval", metrics)
+
+    if training_args.do_predict:
+        logger.info("*** Predict ***")
+
+        # Loop to handle MNLI double evaluation (matched, mis-matched)
+        tasks = [data_args.task_name]
+        predict_datasets = [predict_dataset]
+        if data_args.task_name == "mnli":
+            tasks.append("mnli-mm")
+            predict_datasets.append(raw_datasets["test_mismatched"])
+
+        for predict_dataset, task in zip(predict_datasets, tasks):
+            # Removing the `label` columns because it contains -1 and Trainer won't like that.
+            predict_dataset = predict_dataset.remove_columns("label")
+            predictions = trainer.predict(
+                predict_dataset, metric_key_prefix="predict").predictions
+
+            # save probability
+            out_predprob_file = os.path.join(
+                training_args.output_dir, f"predict_results_probs.csv")
+            np.savetxt(out_predprob_file, predictions, delimiter=",")
+
+            # save predictions
+            predictions = np.squeeze(
+                predictions) if is_regression else np.argmax(predictions, axis=1)
+
+            output_predict_file = os.path.join(
+                training_args.output_dir, f"predict_results_{task}.txt")
+            if trainer.is_world_process_zero():
+                with open(output_predict_file, "w") as writer:
+                    logger.info(f"***** Predict results {task} *****")
+                    writer.write("index\tprediction\n")
+                    for index, item in enumerate(predictions):
+                        if is_regression:
+                            writer.write(f"{index}\t{item:3.3f}\n")
+                        else:
+                            item = label_list[item]
+                            writer.write(f"{index}\t{item}\n")
+
+    if training_args.push_to_hub:
+        kwargs = {"finetuned_from": model_args.model_name_or_path,
+                  "tasks": "text-classification"}
+        if data_args.task_name is not None:
+            kwargs["language"] = "en"
+            kwargs["dataset_tags"] = "glue"
+            kwargs["dataset_args"] = data_args.task_name
+            kwargs["dataset"] = f"GLUE {data_args.task_name.upper()}"
+
+        trainer.push_to_hub(**kwargs)
+
+
+def _mp_fn(index):
+    # For xla_spawn (TPUs)
+    main()
+
+
+if __name__ == "__main__":
+    main()