Upload 46 files

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,20 @@
+# Use an official Python runtime
+FROM python:3.8-slim
+
+# Set the working directory
+WORKDIR /app
+
+# Copy the requirements file
+COPY requirements.txt .
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the rest of the app
+COPY . .
+
+# Expose the Flask port
+EXPOSE 5000
+
+# Run the application
+CMD ["python", "app.py"]

README.md

@@ -1,10 +1,11 @@
----
-title: Detect Edits In AI-Generated Text
-emoji: 👁
-colorFrom: blue
-colorTo: indigo
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Detect Edits In AI-Generated Text
+emoji: 👁
+colorFrom: blue
+colorTo: purple
+sdk: docker
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,76 @@
+#https://www.freecodecamp.org/news/how-to-setup-virtual-environments-in-python/
+#https://www.youtube.com/watch?v=qbLc5a9jdXo&ab_channel=CalebCurry
+#https://stackoverflow.com/questions/26368306/export-is-not-recognized-as-an-internal-or-external-command
+#python3 -m venv .venv
+#source .venv/bin/activate
+#
+#pip freeze > requirements.txt
+#$env:FLASK_APP="application.py" #set FLASK_APP=application.py # export FLASK_APP=application.py 
+#set FLASK_ENV=development #export FLASK_ENV=production
+#flask run #flask run --host=0.0.0.0
+
+#pip install torchvision
+
+from flask import Flask, request, jsonify
+from flask_cors import CORS
+import pandas
+from human_text_detect import detect_human_text
+
+app = Flask(__name__)
+CORS(app)
+
+@app.route('/')
+def index():
+    return 'Hello'
+
+@app.route('/detectHumanInAIText/checkText', methods=['POST'])
+def check_text():
+
+    # Get data
+    print('Get data')
+    data = request.get_json()
+    text = data.get('text')
+    model_name = data.get('model')
+    topic = data.get('topic')
+    
+    # Validate data
+    print('Validate data')
+    answer = validate_data(text, model_name, topic)
+    if answer != '':
+        return jsonify({'answer': answer}), 400
+    
+    topic = check_topic(topic)
+    answer = detect_human_text(model_name, topic, text)
+
+    return jsonify({'answer': answer})
+
+def validate_data(text, model_name, topic):
+    if text is None or text == '':
+        return 'Text is missing'
+
+    if model_name is None or model_name == '':
+        return 'Model name is missing'
+
+    if topic is None or topic == '':
+        return 'Topic is missing'
+    
+    if model_name not in ['GPT2XL', 'PHI2']:
+        return f'Model {model_name} not supported'
+    
+    if topic not in ['Characters', 'Locations', 'Nature', 'Video games', 'Series', 'Movies', 'War']:
+        return f'Topic {topic} not supported'
+    
+    return ''
+
+def check_topic(topic):
+    topic_dict = {
+        'Characters': 'characters',
+        'Locations': 'locations',
+        'Nature': 'nature',
+        'Video games': 'video_games_series_movies',
+        'Series': 'video_games_series_movies',
+        'Movies': 'video_games_series_movies',
+        'War': 'war'
+    }
+
+    return topic_dict[topic]

human_text_detect.py

@@ -0,0 +1,169 @@
+import torch
+import pandas as pd
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import logging
+import numpy as np
+import pickle
+from src.DetectLM import DetectLM
+from src.PerplexityEvaluator import PerplexityEvaluator
+from src.PrepareArticles import PrepareArticles #Idan 
+from src.fit_survival_function import fit_per_length_survival_function
+from glob import glob
+import spacy
+import re
+
+
+logging.basicConfig(level=logging.INFO)
+
+
+def read_all_csv_files(pattern):
+    df = pd.DataFrame()
+    print(pattern)
+    for f in glob(pattern):
+        df = pd.concat([df, pd.read_csv(f)])
+    return df
+
+
+def get_survival_function(df, G=101):
+    """
+    Returns a survival function for every sentence length in tokens.
+
+    Args:
+    :df:  data frame with columns 'response' and 'length'
+    :G:   number of interpolation points
+    
+    Return:
+        bivariate function (length, responce) -> (0,1)
+
+    """
+    assert not df.empty
+    value_name = "response" if "response" in df.columns else "logloss"
+
+    df1 = df[~df[value_name].isna()]
+    ll = df1['length']
+    xx1 = df1[value_name]
+    return fit_per_length_survival_function(ll, xx1, log_space=True, G=G)
+
+
+def mark_edits_remove_tags(chunks, tag="edit"):
+    text_chunks = chunks['text']
+    edits = []
+    for i,text in enumerate(text_chunks):
+        chunk_text = re.findall(rf"<{tag}>(.+)</{tag}>", text)
+        if len(chunk_text) > 0:
+            import pdb; pdb.set_trace()
+            chunks['text'][i] = chunk_text[0]
+            chunks['length'][i] -= 2
+            edits.append(True)
+        else:
+            edits.append(False)
+
+    return chunks, edits
+
+def get_null_data(model_name, topic):
+    data = None
+    try:
+        file = open(f'nullData/{model_name}_{topic}.pkl', 'rb')
+        data = pickle.load(file)
+    except:
+        pass
+
+    return data
+
+def get_threshold_obj(model_name, topic):
+    threshold = None
+    try:
+        file = open('threshold_obj.pkl', 'rb')
+        threshold_obj = pickle.load(file)
+        threshold = threshold_obj[model_name][topic]
+    except:
+        pass
+
+    return threshold
+
+def detect_human_text(model_name, topic, text):
+    
+    # Get null data
+    print('Get null data')
+    df_null = get_null_data(model_name, topic)
+    if 'num' in df_null.columns:
+        df_null = df_null[df_null.num > 1]
+    
+    # Get survival function
+        print('Get survival function')
+    pval_functions = get_survival_function(df_null, G=43)
+
+    min_tokens_per_sentence = 10
+    max_tokens_per_sentence = 100
+
+    # Init model
+    print('Init model')
+    lm_name = 'gpt2-xl' if model_name == 'GPT2XL' else 'microsoft/phi-2'
+    tokenizer = AutoTokenizer.from_pretrained(lm_name)
+    model = AutoModelForCausalLM.from_pretrained(lm_name)
+    
+    print('Init PerplexityEvaluator')
+    sentence_detector = PerplexityEvaluator(model, tokenizer)
+
+    if torch.backends.mps.is_available():
+        device = 'mps'
+    elif torch.cuda.is_available():
+        device = 'cuda'
+    else:
+        device = 'cpu'
+
+    print(f'device {device}')
+    model.to(device)
+
+    print('Init DetectLM')
+    detector = DetectLM(sentence_detector, pval_functions,
+                        min_len=min_tokens_per_sentence,
+                        max_len=max_tokens_per_sentence,
+                        length_limit_policy='truncate',
+                        HC_type='stbl',
+                        ignore_first_sentence= False
+                        )
+
+    # Convert text to object
+    print('Analyze text')
+    article_obj = get_article_obj(text)
+    parser = PrepareArticles(article_obj, min_tokens=min_tokens_per_sentence, max_tokens=max_tokens_per_sentence)
+    chunks = parser(combined=False)
+
+    # Go over all the document
+    for i in range(len(chunks['text'])):
+        print(chunks['text'][i])
+        # for p,v in enumerate(chunks['text'][i]):
+        #     print(f'{p}: {v}')
+        res = detector(chunks['text'][i], chunks['context'][i], dashboard=None)
+
+        # print(f"Num of Edits (rate) = {np.sum(df['tag'] == '<edit>')} ({edit_rate})")
+        # print(f"HC = {res['HC']}")
+        # print(f"Fisher = {res['fisher']}")
+        # print(f"Fisher (chisquared pvalue) = {res['fisher_pvalue']}")
+
+        results = res['HC']
+    
+    threshold = get_threshold_obj(model_name, topic)
+    print(f"threshold: {threshold}, results: {results}")
+    return '1' if results >= threshold else '0'
+
+# Convert article text into object
+def get_article_obj(text):
+    # Init article object
+    article_obj = {
+        'sub_titles': [{
+            'sentences': []
+        }]
+    }
+    
+    nlp = spacy.load("en_core_web_sm")  # Load model
+
+    for line in text.split('\n'):
+        doc = nlp(line) # Analyze text
+        sentences = [sent.text for sent in doc.sents if len(sent) >= 10] # Split it by sentence
+        for sentence in sentences:
+            sentence = re.sub(r' +', ' ', sentence) # Remove duplicate spaces
+            article_obj['sub_titles'][0]['sentences'].append({'sentence': sentence})
+
+    return article_obj

nullData/GPT2XL_characters.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75c6c7b757dd7db42e73ae3fea662d5fc871be22d66b2784531c8996e3dfacc7
+size 3168919

nullData/GPT2XL_locations.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6bba9cb0f09b801a43f1c2bfb04f30b9764ed106d7488db7d44abc207579bb6
+size 3137467

nullData/GPT2XL_nature.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbaab4e41c90faf4c4c8cd794b99045947a7aef5b19a65ed6ec2e0678673cd81
+size 3192531

nullData/GPT2XL_video_games_series_movies.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aabd9d0e5fcb4dc98fbec83c6064df0fb168a172c078ef015afaebc0b1e54e39
+size 3266168

nullData/GPT2XL_war.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15805a913be9c9bb34daf6ce47b011b1f8388b708a0435cd23bf5efe886ebf37
+size 3253367

nullData/PHI2_characters.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:585b56afdca713f6d1b152e69aeef58aa66abd5986c0d05363016b571568e2c1
+size 3168919

nullData/PHI2_locations.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09b0801020a2be2ac32355e38ba6efc4b7a6c5bfa2ad3677d2e0fcda56b54cf1
+size 3137467

nullData/PHI2_nature.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b5b787d4cc5f74b882d064a5d58f8de3f456268e1121c78e2b4ba5b5db5a6c9
+size 3192531

nullData/PHI2_video_games_series_movies.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b24cb4d919b42f20153e7b481a67e2a1e7079468af231b3e6219c803829184d2
+size 3266168

nullData/PHI2_war.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:389ec634c5854434f65c1087d6384089a307e721436cca51dba061bcb30baccb
+size 3253382

src/DetectLM.py

@@ -0,0 +1,178 @@
+import numpy as np
+import pandas as pd
+from multitest import MultiTest
+from tqdm import tqdm
+import logging
+
+
+def truncae_to_max_no_tokens(text, max_no_tokens):
+    return " ".join(text.split()[:max_no_tokens])
+
+
+class DetectLM(object):
+    def __init__(self, sentence_detection_function, survival_function_per_length,
+                 min_len=4, max_len=100, HC_type="stbl",
+                 length_limit_policy='truncate', ignore_first_sentence=False):
+        """
+        Test for the presence of sentences of irregular origin as reflected by the
+        sentence_detection_function. The test is based on the sentence detection function
+        and the P-values obtained from the survival function of the detector's responses.
+
+        Args:
+        ----
+            :sentence_detection_function:  a function returning the response of the text 
+            under the detector. Typically, the response is a logloss value under some language model.
+            :survival_function_per_length:  survival_function_per_length(l, x) is the probability of the language
+            model to produce a sentence value as extreme as x or more when the sentence s is the input to
+            the detector. The function is defined for every sentence length l.
+            The detector can also recieve a context c, in which case the input is the pair (s, c).
+            :length_limit_policy: When a sentence exceeds ``max_len``, we can:
+                'truncate':  truncate sentence to the maximal length :max_len
+                'ignore':  do not evaluate the response and P-value for this sentence
+                'max_available':  use the logloss function of the maximal available length
+            :ignore_first_sentence:  whether to ignore the first sentence in the document or not. Useful when assuming
+        context of the form previous sentence.
+        """
+
+        self.survival_function_per_length = survival_function_per_length
+        self.sentence_detector = sentence_detection_function
+        self.min_len = min_len
+        self.max_len = max_len
+        self.length_limit_policy = length_limit_policy
+        self.ignore_first_sentence = ignore_first_sentence
+        self.HC_stbl = True if HC_type == 'stbl' else False
+
+    def _logperp(self, sent: str, context=None) -> float:
+        return float(self.sentence_detector(sent, context))
+
+    def _test_sentence(self, sentence: str, context=None):
+        return self._logperp(sentence, context)
+    
+    def _get_length(self, sentence: str):
+        return len(sentence.split())
+
+    def _test_response(self, response: float, length: int):
+        """
+        Args:
+            response:  sentence logloss
+            length:    sentence length in tokens
+
+        Returns:
+          pvals:    P-value of the logloss of the sentence
+          comments: comment on the P-value
+        """
+        if self.min_len <= length:
+            comment = "OK"
+            if length > self.max_len:  # in case length exceeds specifications...
+                if self.length_limit_policy == 'truncate':
+                    length = self.max_len
+                    comment = f"truncated to {self.max_len} tokens"
+                elif self.length_limit_policy == 'ignore':
+                    comment = "ignored (above maximum limit)"
+                    return np.nan, np.nan, comment
+                elif self.length_limit_policy == 'max_available':
+                    comment = "exceeding length limit; resorting to max-available length"
+                    length = self.max_len
+            pval = self.survival_function_per_length(length, response)
+            assert pval >= 0, "Negative P-value. Something is wrong."
+            return dict(response=response, 
+                        pvalue=pval, 
+                        length=length,
+                        comment=comment)
+        else:
+            comment = "ignored (below minimal length)"
+            return dict(response=response, 
+                        pvalue=np.nan, 
+                        length=length,
+                        comment=comment)
+
+    def _get_pvals(self, responses: list, lengths: list) -> tuple:
+        pvals = []
+        comments = []
+        for response, length in zip(responses, lengths):
+            r = self._test_response(response, length)
+            pvals.append(float(r['pvalue']))
+            comments.append(r['comment'])
+        return pvals, comments
+
+
+    def _get_responses(self, sentences: list, contexts: list) -> list:
+        """
+        Compute response and length of a text sentence 
+        """
+        assert len(sentences) == len(contexts)
+
+        responses = []
+        lengths = []
+        for sent, ctx in tqdm(zip(sentences, contexts)):
+            logging.debug(f"Testing sentence: {sent} | context: {ctx}")
+            length = self._get_length(sent)
+            if self.length_limit_policy == 'truncate':
+                sent = truncae_to_max_no_tokens(sent, self.max_len)
+            if length == 1:
+                logging.warning(f"Sentence {sent} is too short. Skipping.")
+                responses.append(np.nan)
+                continue
+            try:
+                responses.append(self._test_sentence(sent, ctx))
+            except:
+                # something unusual happened...
+                import pdb; pdb.set_trace()
+            lengths.append(length)
+        return responses, lengths
+
+    def get_pvals(self, sentences: list, contexts: list) -> tuple:
+        """
+        logloss test of every (sentence, context) pair
+        """
+        assert len(sentences) == len(contexts)
+
+        responses, lengths = self._get_responses(sentences, contexts)
+        pvals, comments = self._get_pvals(responses, lengths)
+        
+        return pvals, responses, comments
+
+
+    def testHC(self, sentences: list) -> float:
+        pvals = np.array(self.get_pvals(sentences)[1])
+        mt = MultiTest(pvals, stbl=self.HC_stbl)
+        return mt.hc(gamma=0.4)[0]
+
+    def testFisher(self, sentences: list) -> dict:
+        pvals = np.array(self.get_pvals(sentences)[1])
+        print(pvals)
+        mt = MultiTest(pvals, stbl=self.HC_stbl)
+        return dict(zip(['Fn', 'pvalue'], mt.fisher()))
+
+    def _test_chunked_doc(self, lo_chunks: list, lo_contexts: list) -> tuple:
+        pvals, responses, comments = self.get_pvals(lo_chunks, lo_contexts)
+        if self.ignore_first_sentence:
+            pvals[0] = np.nan
+            logging.info('Ignoring the first sentence.')
+            comments[0] = "ignored (first sentence)"
+        
+        df = pd.DataFrame({'sentence': lo_chunks, 'response': responses, 'pvalue': pvals,
+                           'context': lo_contexts, 'comment': comments},
+                          index=range(len(lo_chunks)))
+        df_test = df[~df.pvalue.isna()]
+        if df_test.empty:
+            logging.warning('No valid chunks to test.')
+            return None, df
+        return MultiTest(df_test.pvalue, stbl=self.HC_stbl), df
+
+    def test_chunked_doc(self, lo_chunks: list, lo_contexts: list, dashboard=False) -> dict:
+        mt, df = self._test_chunked_doc(lo_chunks, lo_contexts)
+        if mt is None:
+            hc = np.nan
+            fisher = (np.nan, np.nan)
+            df['mask'] = pd.NA
+        else:
+            hc, hct = mt.hc(gamma=0.4)
+            fisher = mt.fisher()
+            df['mask'] = df['pvalue'] <= hct
+        if dashboard:
+            mt.hc_dashboard(gamma=0.4)
+        return dict(sentences=df, HC=hc, fisher=fisher[0], fisher_pvalue=fisher[1])
+
+    def __call__(self, lo_chunks: list, lo_contexts: list, dashboard=False) -> dict:
+        return self.test_chunked_doc(lo_chunks, lo_contexts, dashboard=dashboard)

src/HC_survival_function.py

@@ -0,0 +1,66 @@
+"""
+This script computes the survival function of the HC statistic for a given sample size n.
+The survival function is computed using a simulation of the null distribution of the HC statistic.
+We use the simulation results to fit a bivariate function of the form Pr[HC >= x | n] = f(n, x).
+The simulation results are saved in a file named HC_null_sim_results.csv.
+use function get_HC_survival_function to load the bivariate function or simulate the distribution. 
+"""
+
+import numpy as np
+import pandas as pd
+from multitest import MultiTest
+from tqdm import tqdm
+from scipy.interpolate import RectBivariateSpline
+from src.fit_survival_function import fit_survival_func
+import logging
+
+HC_NULL_SIM_FILE = "HC_null_sim_results.csv"
+STBL = True
+NN = [25, 50, 75, 100, 125, 150, 200, 250, 300, 400, 500]  # values of n to simulate
+
+def get_HC_survival_function(HC_null_sim_file, log_space=True, nMonte=10000, STBL=True):
+
+    xx = {}
+    if HC_null_sim_file is None:            
+            logging.info("Simulated HC null values file was not provided.")
+            for n in tqdm(NN):
+                logging.info(f"Simulating HC null values for n={n}...")
+                yy = np.zeros(nMonte)
+                for j in range(nMonte):
+                    uu = np.random.rand(n)
+                    mt = MultiTest(uu, stbl=STBL)
+                    yy[j] = mt.hc()[0]
+                xx[n] = yy
+            nn = NN # Idan
+    else:
+        logging.info(f"Loading HC null values from {HC_null_sim_file}...")
+        df = pd.read_csv(HC_null_sim_file, index_col=0)
+        for n in df.index:
+            xx[n] = df.loc[n]
+        nn = df.index.tolist()
+
+    xx0 = np.linspace(-1, 10, 57)
+    zz = []
+    for n in nn:
+        univariate_survival_func = fit_survival_func(xx[n], log_space=log_space)
+        zz.append(univariate_survival_func(xx0))
+        
+    func_log = RectBivariateSpline(np.array(nn), xx0, np.vstack(zz))
+
+    if log_space:
+        def func(x, y):
+            return np.exp(-func_log(x,y))
+        return func
+    else:
+        return func_log
+    
+
+def main():
+    func = get_HC_survival_function(HC_null_sim_file=HC_NULL_SIM_FILE, STBL=STBL)
+    print("Pr[HC >= 3 |n=50] = ", func(50, 3)[0][0]) # 9.680113e-05
+    print("Pr[HC >= 3 |n=100] = ", func(100, 3)[0][0]) # 0.0002335
+    print("Pr[HC >= 3 |n=200] = ", func(200, 3)[0][0]) # 0.00103771
+    
+
+if __name__ == '__main__':
+    main()

src/PerplexityEvaluator.py

@@ -0,0 +1,34 @@
+import torch
+
+class PerplexityEvaluator(object):
+    def __init__(self, model, tokenizer, ignore_index=-1):
+        self.model = model
+        self.tokenizer = tokenizer
+        self.ignore_index = ignore_index
+
+    def __call__(self, text, context=None):
+        return self.log_perplexity(text, context)
+
+    def log_perplexity(self, text, context=None):
+        """
+        Evaluate log perplexity of text with respect to the language model
+        based on the context
+
+        :param text:
+        :param context:
+        :return:
+        """
+        device = self.model.device
+        text_ids = self.tokenizer(text, return_tensors='pt')
+        if context:
+            context_ids = self.tokenizer(context, return_tensors='pt')
+            input_ids = torch.concatenate([context_ids['input_ids'], text_ids['input_ids']], axis=1)
+            labels = torch.concatenate([torch.ones_like(context_ids['input_ids']) * self.ignore_index,
+                                        text_ids['input_ids']], axis=1)
+            print("Warning, need to remove context length when reporting lppx")
+        else:
+            input_ids = text_ids['input_ids']
+            labels = input_ids
+
+        loss = self.model(input_ids=input_ids.to(device), labels=labels.to(device)).loss
+        return loss.cpu().detach().numpy()

src/PrepareArticles.py

@@ -0,0 +1,74 @@
+import pandas as pd
+import numpy as np
+import json
+
+class PrepareArticles(object):
+    """
+    Parse preprocessed data from csv
+
+    This information is needed for evaluating log-perplexity of the text with respect to a language model
+    and later on to test the likelihood that the sentence was sampled from the model with the relevant context.
+    """
+    def __init__(self, article_obj, get_edits=False, min_tokens=10, max_tokens=100, max_sentences=None):
+        self.article_obj = article_obj
+        self.min_tokens = min_tokens
+        self.max_tokens = max_tokens
+        self.get_edits = get_edits
+        self.max_sentences = max_sentences
+
+    def __call__(self, combined=True):
+        return self.parse_dataset(combined)
+    
+    def parse_dataset(self, combined=True):
+
+        texts = []
+        lengths = []
+        contexts = []
+        tags = []
+        
+        current_texts = []
+        current_lengths = []
+        current_contexts = []
+        current_tags = []
+        exceeded_max_sentences = False
+        
+        for sub_title in self.article_obj['sub_titles']: # For each sub title
+            for sentence in sub_title['sentences']: # Go over each sentence
+                sentence_size = len(sentence['sentence'].split())
+                if sentence_size >= self.min_tokens and sentence_size <= self.max_tokens:
+                    current_texts.append(sentence['sentence'])
+                    current_lengths.append(len(sentence['sentence'].split())) # Number of tokens
+                    current_contexts.append(sentence['context'] if 'context' in sentence else None)
+                    current_tags.append('no edits')
+
+                # If get_edits and has edited sentence save it
+                if self.get_edits and 'alternative' in sentence and len(sentence['alternative'].split()) >= self.min_tokens and len(sentence['alternative'].split()) <= self.max_tokens:
+                    current_texts.append(sentence['alternative'])
+                    current_lengths.append(len(sentence['alternative'].split()))
+                    current_contexts.append(sentence['alternative_context'] if 'alternative_context' in sentence else None)
+                    current_tags.append('<edit>')
+                if self.max_sentences and len(current_texts) >= self.max_sentences:
+                    exceeded_max_sentences = True
+                    break
+                    # return {'text': np.array(texts, dtype=object), 'length': np.array(lengths, dtype=object), 'context': np.array(contexts, dtype=object), 'tag': np.array(tags, dtype=object),
+                    #             'number_in_par': np.arange(1,1+len(texts))}
+            if exceeded_max_sentences:
+                break
+        
+        # If exceede max sentences only if self.max_sentences is not None
+        if (self.max_sentences and exceeded_max_sentences) or (not self.max_sentences):
+            # If combined, combine the data
+            if combined:
+                texts = texts + current_texts
+                lengths = lengths + current_lengths
+                contexts = contexts + current_contexts
+                tags = tags + current_tags
+            else:
+                texts.append(np.array(current_texts))
+                lengths.append(np.array(current_lengths))
+                contexts.append(np.array(current_contexts))
+                tags.append(np.array(current_tags))
+            
+        return {'text': np.array(texts, dtype=object), 'length': np.array(lengths, dtype=object), 'context': np.array(contexts, dtype=object), 'tag': np.array(tags, dtype=object),
+                    'number_in_par': np.arange(1,1+len(texts))}
+

src/PrepareSentenceContext.py

@@ -0,0 +1,158 @@
+import logging
+import spacy
+import re
+import numpy as np
+from src.SentenceParser import SentenceParser
+
+class PrepareSentenceContext(object):
+    """
+    Parse text and extract length and context information
+
+    This information is needed for evaluating log-perplexity of the text with respect to a language model
+    and later on to test the likelihood that the sentence was sampled from the model with the relevant context.
+    """
+
+    def __init__(self, sentence_parser='spacy', context_policy=None, context=None):
+        if sentence_parser == 'spacy':
+            self.nlp = spacy.load("en_core_web_sm", disable=["tagger", "attribute_ruler", "lemmatizer", "ner"])
+        if sentence_parser == 'regex':
+            logging.warning("Regex-based parser is not good at breaking sentences like 'Dr. Stone', etc.")
+            self.nlp = SentenceParser()
+            
+        self.sentence_parser_name = sentence_parser
+
+        self.context_policy = context_policy
+        self.context = context
+
+    def __call__(self, text):
+        return self.parse_sentences(text)
+
+    def parse_sentences(self, text):
+        pattern_close = r"(.*?)</edit>"
+        pattern_open = r"<edit>(.*?)"
+        MIN_TOKEN_LEN = 3
+
+        texts = []
+        tags = []
+        lengths = []
+        contexts = []
+
+        def update_sent(sent_text, tag, sent_length):
+            texts.append(sent_text)
+            tags.append(tag)
+            lengths.append(sent_length)
+            if self.context is not None:
+                context = self.context
+            elif self.context_policy is None:
+                context = None
+            elif self.context_policy == 'previous_sentence' and len(texts) > 0:
+                context = texts[-1]
+            else:
+                context = None
+            contexts.append(context)
+
+        curr_tag = None
+        parsed = self.nlp(text)
+        for s in parsed.sents:
+            prev_tag = curr_tag
+            matches_close = re.findall(pattern_close, s.text)
+            matches_open = re.findall(pattern_open, s.text)
+            matches_between = re.findall(r"<edit>(.*?)</edit>", s.text)
+            
+            logging.debug(f"Current sentence: {s.text}")
+            logging.debug(f"Matches open: {matches_open}")
+            logging.debug(f"Matches close: {matches_close}")
+            logging.debug(f"Matches between: {matches_between}")
+            if len(matches_close)>0 and len(matches_open)>0: 
+                logging.debug("Found an opening and a closing tag in the same sentence.")
+                if prev_tag is None and len(matches_open[0]) >= MIN_TOKEN_LEN:
+                    logging.debug("Openning followed by closing with some text in between.")
+                    update_sent(matches_open[0], "<edit>", len(s)-2)
+                    curr_tag = None
+                if prev_tag == "<edit>" and len(matches_close[0]) >= MIN_TOKEN_LEN:
+                    logging.warning(f"Wierd case: closing/openning followed by openning in sentence {len(texts)}")
+                    update_sent(matches_close[0], prev_tag, len(s)-1)
+                    curr_tag = None
+                if prev_tag == "</edit>":
+                    logging.debug("Closing followed by openning.")
+                    curr_tag = "<edit>"
+                    if len(matches_between[0]) > MIN_TOKEN_LEN:
+                        update_sent(matches_between[0], None, len(s)-2)
+            elif len(matches_open) > 0:
+                curr_tag = "<edit>"
+                assert prev_tag is None, f"Found an opening tag without a closing tag in sentence num. {len(texts)}"
+                if len(matches_open[0]) >= MIN_TOKEN_LEN:
+                    # text and tag are in the same sentence
+                    sent_text = matches_open[0]
+                    update_sent(sent_text, curr_tag, len(s)-1)      
+            elif len(matches_close) > 0:
+                curr_tag = "</edit>"
+                assert prev_tag == "<edit>", f"Found a closing tag without an opening tag in sentence num. {len(texts)}"
+                if len(matches_close[0]) >= MIN_TOKEN_LEN:
+                    # text and tag are in the same sentence
+                    update_sent(matches_close[0], prev_tag, len(s)-1)
+                curr_tag = None
+            else:
+                #if len(matches_close)==0 and len(matches_open)==0: 
+                # no tag
+                update_sent(s.text, curr_tag, len(s))
+        return {'text': texts, 'length': lengths, 'context': contexts, 'tag': tags,
+                    'number_in_par': np.arange(1,1+len(texts))}
+
+    def REMOVE_parse_sentences(self, text):
+        texts = []
+        contexts = []
+        lengths = []
+        tags = []
+        num_in_par = []
+        previous = None
+
+        text = re.sub("(</?[a-zA-Z0-9 ]+>\.?)\s+", r"\1.\n", text)  # to make sure that tags are in separate sentences
+        #text = re.sub("(</[a-zA-Z0-9 ]+>\.?)\s+", r"\n\1.\n", text)  # to make sure that tags are in separate sentences
+
+        parsed = self.nlp(text)
+
+        running_sent_num = 0
+        curr_tag = None
+        for i, sent in enumerate(parsed.sents):
+            # Here we try to track HTML-like tags. There might be
+            # some issues because spacy sentence parser has unexpected behavior when it comes to newlines
+            all_tags = re.findall(r"(</?[a-zA-Z0-9 ]+>)", str(sent))
+            if len(all_tags) > 1:
+                    logging.error(f"More than one tag in sentence {i}: {all_tags}")
+                    exit(1)
+            if len(all_tags) == 1:
+                tag = all_tags[0]
+                if tag[:2] == '</': # a closing tag
+                    if curr_tag is None:
+                        logging.warning(f"Closing tag without an opening tag in sentence {i}: {sent}")
+                    else:
+                        curr_tag = None
+                else:
+                    if curr_tag is not None:
+                        logging.warning(f"Opening tag without a closing tag in sentence {i}: {sent}")
+                    else:
+                        curr_tag = tag
+            else:  # if text is not a tag
+                sent_text = str(sent)
+                sent_length = len(sent)
+
+                texts.append(sent_text)
+                running_sent_num += 1
+                num_in_par.append(running_sent_num)
+                tags.append(curr_tag)
+                lengths.append(sent_length)
+
+                if self.context is not None:
+                    context = self.context
+                elif self.context_policy is None:
+                    context = None
+                elif self.context_policy == 'previous_sentence':
+                    context = previous
+                    previous = sent_text
+                else:
+                    context = None
+
+                contexts.append(context)
+        return {'text': texts, 'length': lengths, 'context': contexts, 'tag': tags,
+                'number_in_par': num_in_par}

src/SentenceParser.py

@@ -0,0 +1,31 @@
+import re
+
+
+class Sentence(object):
+    def __init__(self, text):
+        self.text = text
+        self.tokens = text.split()
+
+    def __len__(self):
+        return len(self.tokens)
+
+class Sentences(object):
+    def __init__(self, text):
+        def iterate(text):
+            for s in re.split(r"\n", text):
+                yield s
+        self.sents = iterate(text)
+
+    def __len__(self):
+        return len(self.sents)
+
+class SentenceParser(object):
+    """
+    Iterate over the text column of a dataframe
+    """
+
+    def __init__(self):
+        self.sents = None
+
+    def __call__(self, text):
+        return Sentences(text)

src/dataset_loaders.py

@@ -0,0 +1,87 @@
+from datasets import load_dataset
+
+SEED = 42
+
+
+def get_dataset(name: str, machine_field, human_field, iterable=False,
+                text_field=None, shuffle=False, main_split='train'):
+    dataset = load_dataset(name)[main_split]
+    ds = dataset.rename_columns({human_field: 'human_text', machine_field: 'machine_text'})
+    if 'id' not in ds.features:
+        ids = list(range(len(ds)))
+        ds = ds.add_column("id", ids)
+    if text_field:
+        ds = ds.rename_columns({text_field: 'text'})
+
+    if iterable:
+        ds = ds.to_iterable_dataset()
+    if shuffle:
+        return ds.shuffle(seed=SEED)
+    else:
+        return ds
+
+
+def get_text_from_wiki_dataset(shuffle=False, text_field=None):
+    return get_dataset(name="aadityaubhat/GPT-wiki-intro", machine_field='generated_intro',
+                       human_field="wiki_intro", shuffle=shuffle, text_field=text_field)
+
+
+def get_text_from_wiki_long_dataset(shuffle=False, text_field=None):
+    return get_dataset(name="alonkipnis/wiki-intro-long", machine_field='generated_intro',
+                       human_field="wiki_intro", shuffle=shuffle, text_field=text_field)
+
+
+def get_text_from_wiki_long_dataset_local(shuffle=False, text_field=None, iterable=False):
+    """
+    A version of wiki_intro dataset with at least 15 sentences per generated article
+    """
+    dataset = load_dataset("alonkipnis/wiki-intro-long")
+    ds = dataset.rename_columns({"wiki_intro": 'human_text', "generated_intro": 'machine_text'})
+    if text_field:
+        ds = ds.rename_columns({text_field: 'text'})
+    if iterable:
+        ds = ds.to_iterable_dataset()
+    if shuffle:
+        return ds.shuffle(seed=SEED)
+    else:
+        return ds
+
+
+def get_text_from_chatgpt_news_long_dataset_local(shuffle=False, text_field=None, iterable=False):
+    """
+    A version of chatgpt-news-articles dataset with at least 15 sentences per generated article
+    Only 'train' split is included
+    """
+    dataset = load_dataset("alonkipnis/news-chatgpt-long")
+    ds = dataset.rename_columns({"article": 'human_text', "chatgpt": 'machine_text'})
+    if text_field:
+        ds = ds.rename_columns({text_field: 'text'})
+    if iterable:
+        ds = ds.to_iterable_dataset()
+    if shuffle:
+        return ds.shuffle(seed=SEED)
+    else:
+        return ds
+
+def get_text_from_chatgpt_abstracts_dataset(shuffle=False, text_field=None):
+    return get_dataset(name="NicolaiSivesind/ChatGPT-Research-Abstracts", machine_field="generated_abstract",
+                       human_field="real_abstract", shuffle=shuffle, text_field=text_field)
+
+def get_text_from_chatgpt_news_long_dataset(shuffle=False, text_field=None):
+    return get_dataset(name="alonkipnis/news-chatgpt-long", machine_field='chatgpt',
+                       human_field="article", shuffle=shuffle, text_field=text_field)
+
+
+def get_text_from_chatgpt_news_dataset(shuffle=False, text_field=None):
+    return get_dataset(name="isarth/chatgpt-news-articles", machine_field='chatgpt',
+                       human_field="article", shuffle=shuffle, text_field=text_field)
+
+
+def get_text_from_wikibio_dataset(shuffle=False, text_field=None):
+    return get_dataset(name="potsawee/wiki_bio_gpt3_hallucination", machine_field='gpt3_text',
+                       human_field="wiki_bio_text", shuffle=shuffle, text_field=text_field, main_split='evaluation')
+
+## New datasets (22/5/2023)
+def get_text_from_alpaca_gpt4_dataset(shuffle=False, text_field=None):
+    return get_dataset(name="polyware-ai/alpaca-gpt4-cleaned", machine_field='output',
+                       human_field="instruction", shuffle=shuffle, text_field=text_field)

src/fit_survival_function.py

@@ -0,0 +1,94 @@
+"""
+Script to read log-loss data of many sentences and characterize the empirical distribution.
+We also report the mean log-loss as a function of sentence length
+"""
+from scipy.interpolate import RectBivariateSpline, interp1d
+import numpy as np
+
+def fit_survival_func(xx, log_space=True):
+    """
+    Returns an estimated survival function to the data in :xx: using
+    interpolation.
+
+    Args:
+        :xx:  data
+        :log_space:  indicates whether fitting is in log space or not.
+
+    Returns:
+         univariate function
+    """
+    assert len(xx) > 0
+
+    eps = 1 / len(xx)
+    inf = 1 / eps
+
+    sxx = np.sort(xx)
+    qq = np.mean(np.expand_dims(sxx,1) >= sxx, 0)
+
+    if log_space:
+        qq = -np.log(qq)
+
+
+    if log_space:
+        return interp1d(sxx, qq, fill_value=(0 , np.log(inf)), bounds_error=False)
+    else:
+        return interp1d(sxx, qq, fill_value=(1 , 0), bounds_error=False)
+
+
+def fit_per_length_survival_function(lengths, xx, G=501, log_space=True):
+    """
+    Returns a survival function for every sentence length in tokens.
+    Use 2D interpolation over the empirical survival function of the pairs (length, x)
+    
+    Args:
+        :lengths:, :xx:, 1-D arrays
+        :G:  number of grid points to use in the interpolation in the xx dimension
+        :log_space:  indicates whether result is in log space or not.
+
+    Returns:
+        bivariate function (length, x) -> [0,1]
+    """
+
+    assert len(lengths) == len(xx)
+
+    min_tokens_per_sentence = lengths.min()
+    max_tokens_per_sentence = lengths.max()
+    ll = np.arange(min_tokens_per_sentence, max_tokens_per_sentence)
+
+    ppx_min_val = xx.min()
+    ppx_max_val = xx.max()
+    xx0 = np.linspace(ppx_min_val, ppx_max_val, G)
+
+    ll_valid = []
+    zz = []
+    for l in ll:
+        xx1 = xx[lengths == l]
+        if len(xx1) > 1:
+            univariate_survival_func = fit_survival_func(xx1, log_space=log_space)
+            ll_valid.append(l)
+            zz.append(univariate_survival_func(xx0))
+
+    func = RectBivariateSpline(np.array(ll_valid), xx0, np.vstack(zz))
+    if log_space:
+        def func2d(x, y):
+            return np.exp(-func(x,y))
+        return func2d
+    else:
+        return func
+    
+
+# import pickle
+# import pandas as pd
+# df = pd.read_csv('D:\\.Idan\\תואר שני\\תזה\\detectLM\\article_null.csv')
+# LOGLOSS_PVAL_FUNC_FILE = 'D:\.Idan\תואר שני\תזה\detectLM\example\logloss_pval_function.pkl'
+# LOGLOSS_PVAL_FUNC_FILE_TEST = 'D:\.Idan\תואר שני\תזה\detectLM\example\logloss_pval_function_test.pkl'
+# with open(LOGLOSS_PVAL_FUNC_FILE, 'wb') as handle:
+#     pickle.dump(fit_per_length_survival_function(df['length'].values, df['response'].values), handle, protocol=pickle.HIGHEST_PROTOCOL)
+
+# with open(LOGLOSS_PVAL_FUNC_FILE, 'rb') as f:
+#     data = pickle.load(f)
+#     print(data)
+
+# with open(LOGLOSS_PVAL_FUNC_FILE_TEST, 'rb') as f:
+#     data = pickle.load(f)
+#     print(data)

threshold_obj.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b9b1dfc4fc552a4c975ebe1f05a5140bee30fc8231fd4b4eba1dcf4082d127a
+size 208