Delete Detect_Edits_in_AI-Generated_Text

Detect_Edits_in_AI-Generated_Text/.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

Detect_Edits_in_AI-Generated_Text/Dockerfile

@@ -1,20 +0,0 @@
-# Use an official Python runtime
-FROM python:3.8-slim
-
-# Set the working directory
-WORKDIR /app
-
-# Copy the requirements file
-COPY requirements.txt .
-
-# Install dependencies
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Copy the rest of the app
-COPY . .
-
-# Expose the Flask port
-EXPOSE 5000
-
-# Run the application
-CMD ["python", "app.py"]

Detect_Edits_in_AI-Generated_Text/README.md

@@ -1,11 +0,0 @@
----
-title: Detect Edits In AI-Generated Text
-emoji: 👁
-colorFrom: blue
-colorTo: purple
-sdk: docker
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

Detect_Edits_in_AI-Generated_Text/app.py

@@ -1,76 +0,0 @@
-#https://www.freecodecamp.org/news/how-to-setup-virtual-environments-in-python/
-#https://www.youtube.com/watch?v=qbLc5a9jdXo&ab_channel=CalebCurry
-#https://stackoverflow.com/questions/26368306/export-is-not-recognized-as-an-internal-or-external-command
-#python3 -m venv .venv
-#source .venv/bin/activate
-#
-#pip freeze > requirements.txt
-#$env:FLASK_APP="application.py" #set FLASK_APP=application.py # export FLASK_APP=application.py 
-#set FLASK_ENV=development #export FLASK_ENV=production
-#flask run #flask run --host=0.0.0.0
-
-#pip install torchvision
-
-from flask import Flask, request, jsonify
-from flask_cors import CORS
-import pandas
-from human_text_detect import detect_human_text
-
-app = Flask(__name__)
-CORS(app)
-
-@app.route('/')
-def index():
-    return 'Hello'
-
-@app.route('/detectHumanInAIText/checkText', methods=['POST'])
-def check_text():
-
-    # Get data
-    print('Get data')
-    data = request.get_json()
-    text = data.get('text')
-    model_name = data.get('model')
-    topic = data.get('topic')
-    
-    # Validate data
-    print('Validate data')
-    answer = validate_data(text, model_name, topic)
-    if answer != '':
-        return jsonify({'answer': answer}), 400
-    
-    topic = check_topic(topic)
-    answer = detect_human_text(model_name, topic, text)
-
-    return jsonify({'answer': answer})
-
-def validate_data(text, model_name, topic):
-    if text is None or text == '':
-        return 'Text is missing'
-
-    if model_name is None or model_name == '':
-        return 'Model name is missing'
-
-    if topic is None or topic == '':
-        return 'Topic is missing'
-    
-    if model_name not in ['GPT2XL', 'PHI2']:
-        return f'Model {model_name} not supported'
-    
-    if topic not in ['Characters', 'Locations', 'Nature', 'Video games', 'Series', 'Movies', 'War']:
-        return f'Topic {topic} not supported'
-    
-    return ''
-
-def check_topic(topic):
-    topic_dict = {
-        'Characters': 'characters',
-        'Locations': 'locations',
-        'Nature': 'nature',
-        'Video games': 'video_games_series_movies',
-        'Series': 'video_games_series_movies',
-        'Movies': 'video_games_series_movies',
-        'War': 'war'
-    }
-
-    return topic_dict[topic]

Detect_Edits_in_AI-Generated_Text/human_text_detect.py

@@ -1,169 +0,0 @@
-import torch
-import pandas as pd
-from transformers import AutoTokenizer, AutoModelForCausalLM
-import logging
-import numpy as np
-import pickle
-from src.DetectLM import DetectLM
-from src.PerplexityEvaluator import PerplexityEvaluator
-from src.PrepareArticles import PrepareArticles #Idan 
-from src.fit_survival_function import fit_per_length_survival_function
-from glob import glob
-import spacy
-import re
-
-
-logging.basicConfig(level=logging.INFO)
-
-
-def read_all_csv_files(pattern):
-    df = pd.DataFrame()
-    print(pattern)
-    for f in glob(pattern):
-        df = pd.concat([df, pd.read_csv(f)])
-    return df
-
-
-def get_survival_function(df, G=101):
-    """
-    Returns a survival function for every sentence length in tokens.
-
-    Args:
-    :df:  data frame with columns 'response' and 'length'
-    :G:   number of interpolation points
-    
-    Return:
-        bivariate function (length, responce) -> (0,1)
-
-    """
-    assert not df.empty
-    value_name = "response" if "response" in df.columns else "logloss"
-
-    df1 = df[~df[value_name].isna()]
-    ll = df1['length']
-    xx1 = df1[value_name]
-    return fit_per_length_survival_function(ll, xx1, log_space=True, G=G)
-
-
-def mark_edits_remove_tags(chunks, tag="edit"):
-    text_chunks = chunks['text']
-    edits = []
-    for i,text in enumerate(text_chunks):
-        chunk_text = re.findall(rf"<{tag}>(.+)</{tag}>", text)
-        if len(chunk_text) > 0:
-            import pdb; pdb.set_trace()
-            chunks['text'][i] = chunk_text[0]
-            chunks['length'][i] -= 2
-            edits.append(True)
-        else:
-            edits.append(False)
-
-    return chunks, edits
-
-def get_null_data(model_name, topic):
-    data = None
-    try:
-        file = open(f'nullData/{model_name}_{topic}.pkl', 'rb')
-        data = pickle.load(file)
-    except:
-        pass
-
-    return data
-
-def get_threshold_obj(model_name, topic):
-    threshold = None
-    try:
-        file = open('threshold_obj.pkl', 'rb')
-        threshold_obj = pickle.load(file)
-        threshold = threshold_obj[model_name][topic]
-    except:
-        pass
-
-    return threshold
-
-def detect_human_text(model_name, topic, text):
-    
-    # Get null data
-    print('Get null data')
-    df_null = get_null_data(model_name, topic)
-    if 'num' in df_null.columns:
-        df_null = df_null[df_null.num > 1]
-    
-    # Get survival function
-        print('Get survival function')
-    pval_functions = get_survival_function(df_null, G=43)
-
-    min_tokens_per_sentence = 10
-    max_tokens_per_sentence = 100
-
-    # Init model
-    print('Init model')
-    lm_name = 'gpt2-xl' if model_name == 'GPT2XL' else 'microsoft/phi-2'
-    tokenizer = AutoTokenizer.from_pretrained(lm_name)
-    model = AutoModelForCausalLM.from_pretrained(lm_name)
-    
-    print('Init PerplexityEvaluator')
-    sentence_detector = PerplexityEvaluator(model, tokenizer)
-
-    if torch.backends.mps.is_available():
-        device = 'mps'
-    elif torch.cuda.is_available():
-        device = 'cuda'
-    else:
-        device = 'cpu'
-
-    print(f'device {device}')
-    model.to(device)
-
-    print('Init DetectLM')
-    detector = DetectLM(sentence_detector, pval_functions,
-                        min_len=min_tokens_per_sentence,
-                        max_len=max_tokens_per_sentence,
-                        length_limit_policy='truncate',
-                        HC_type='stbl',
-                        ignore_first_sentence= False
-                        )
-
-    # Convert text to object
-    print('Analyze text')
-    article_obj = get_article_obj(text)
-    parser = PrepareArticles(article_obj, min_tokens=min_tokens_per_sentence, max_tokens=max_tokens_per_sentence)
-    chunks = parser(combined=False)
-
-    # Go over all the document
-    for i in range(len(chunks['text'])):
-        print(chunks['text'][i])
-        # for p,v in enumerate(chunks['text'][i]):
-        #     print(f'{p}: {v}')
-        res = detector(chunks['text'][i], chunks['context'][i], dashboard=None)
-
-        # print(f"Num of Edits (rate) = {np.sum(df['tag'] == '<edit>')} ({edit_rate})")
-        # print(f"HC = {res['HC']}")
-        # print(f"Fisher = {res['fisher']}")
-        # print(f"Fisher (chisquared pvalue) = {res['fisher_pvalue']}")
-
-        results = res['HC']
-    
-    threshold = get_threshold_obj(model_name, topic)
-    print(f"threshold: {threshold}, results: {results}")
-    return '1' if results >= threshold else '0'
-
-# Convert article text into object
-def get_article_obj(text):
-    # Init article object
-    article_obj = {
-        'sub_titles': [{
-            'sentences': []
-        }]
-    }
-    
-    nlp = spacy.load("en_core_web_sm")  # Load model
-
-    for line in text.split('\n'):
-        doc = nlp(line) # Analyze text
-        sentences = [sent.text for sent in doc.sents if len(sent) >= 10] # Split it by sentence
-        for sentence in sentences:
-            sentence = re.sub(r' +', ' ', sentence) # Remove duplicate spaces
-            article_obj['sub_titles'][0]['sentences'].append({'sentence': sentence})
-
-    return article_obj

Detect_Edits_in_AI-Generated_Text/nullData/GPT2XL_characters.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:75c6c7b757dd7db42e73ae3fea662d5fc871be22d66b2784531c8996e3dfacc7
-size 3168919

Detect_Edits_in_AI-Generated_Text/nullData/GPT2XL_locations.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f6bba9cb0f09b801a43f1c2bfb04f30b9764ed106d7488db7d44abc207579bb6
-size 3137467

Detect_Edits_in_AI-Generated_Text/nullData/GPT2XL_nature.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:bbaab4e41c90faf4c4c8cd794b99045947a7aef5b19a65ed6ec2e0678673cd81
-size 3192531

Detect_Edits_in_AI-Generated_Text/nullData/GPT2XL_video_games_series_movies.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aabd9d0e5fcb4dc98fbec83c6064df0fb168a172c078ef015afaebc0b1e54e39
-size 3266168

Detect_Edits_in_AI-Generated_Text/nullData/GPT2XL_war.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:15805a913be9c9bb34daf6ce47b011b1f8388b708a0435cd23bf5efe886ebf37
-size 3253367

Detect_Edits_in_AI-Generated_Text/nullData/PHI2_characters.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:585b56afdca713f6d1b152e69aeef58aa66abd5986c0d05363016b571568e2c1
-size 3168919

Detect_Edits_in_AI-Generated_Text/nullData/PHI2_locations.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:09b0801020a2be2ac32355e38ba6efc4b7a6c5bfa2ad3677d2e0fcda56b54cf1
-size 3137467

Detect_Edits_in_AI-Generated_Text/nullData/PHI2_nature.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9b5b787d4cc5f74b882d064a5d58f8de3f456268e1121c78e2b4ba5b5db5a6c9
-size 3192531

Detect_Edits_in_AI-Generated_Text/nullData/PHI2_video_games_series_movies.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b24cb4d919b42f20153e7b481a67e2a1e7079468af231b3e6219c803829184d2
-size 3266168

Detect_Edits_in_AI-Generated_Text/nullData/PHI2_war.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:389ec634c5854434f65c1087d6384089a307e721436cca51dba061bcb30baccb
-size 3253382

Detect_Edits_in_AI-Generated_Text/src/DetectLM.py

@@ -1,178 +0,0 @@
-import numpy as np
-import pandas as pd
-from multitest import MultiTest
-from tqdm import tqdm
-import logging
-
-
-def truncae_to_max_no_tokens(text, max_no_tokens):
-    return " ".join(text.split()[:max_no_tokens])
-
-
-class DetectLM(object):
-    def __init__(self, sentence_detection_function, survival_function_per_length,
-                 min_len=4, max_len=100, HC_type="stbl",
-                 length_limit_policy='truncate', ignore_first_sentence=False):
-        """
-        Test for the presence of sentences of irregular origin as reflected by the
-        sentence_detection_function. The test is based on the sentence detection function
-        and the P-values obtained from the survival function of the detector's responses.
-
-        Args:
-        ----
-            :sentence_detection_function:  a function returning the response of the text 
-            under the detector. Typically, the response is a logloss value under some language model.
-            :survival_function_per_length:  survival_function_per_length(l, x) is the probability of the language
-            model to produce a sentence value as extreme as x or more when the sentence s is the input to
-            the detector. The function is defined for every sentence length l.
-            The detector can also recieve a context c, in which case the input is the pair (s, c).
-            :length_limit_policy: When a sentence exceeds ``max_len``, we can:
-                'truncate':  truncate sentence to the maximal length :max_len
-                'ignore':  do not evaluate the response and P-value for this sentence
-                'max_available':  use the logloss function of the maximal available length
-            :ignore_first_sentence:  whether to ignore the first sentence in the document or not. Useful when assuming
-        context of the form previous sentence.
-        """
-
-        self.survival_function_per_length = survival_function_per_length
-        self.sentence_detector = sentence_detection_function
-        self.min_len = min_len
-        self.max_len = max_len
-        self.length_limit_policy = length_limit_policy
-        self.ignore_first_sentence = ignore_first_sentence
-        self.HC_stbl = True if HC_type == 'stbl' else False
-
-    def _logperp(self, sent: str, context=None) -> float:
-        return float(self.sentence_detector(sent, context))
-
-    def _test_sentence(self, sentence: str, context=None):
-        return self._logperp(sentence, context)
-    
-    def _get_length(self, sentence: str):
-        return len(sentence.split())
-
-    def _test_response(self, response: float, length: int):
-        """
-        Args:
-            response:  sentence logloss
-            length:    sentence length in tokens
-
-        Returns:
-          pvals:    P-value of the logloss of the sentence
-          comments: comment on the P-value
-        """
-        if self.min_len <= length:
-            comment = "OK"
-            if length > self.max_len:  # in case length exceeds specifications...
-                if self.length_limit_policy == 'truncate':
-                    length = self.max_len
-                    comment = f"truncated to {self.max_len} tokens"
-                elif self.length_limit_policy == 'ignore':
-                    comment = "ignored (above maximum limit)"
-                    return np.nan, np.nan, comment
-                elif self.length_limit_policy == 'max_available':
-                    comment = "exceeding length limit; resorting to max-available length"
-                    length = self.max_len
-            pval = self.survival_function_per_length(length, response)
-            assert pval >= 0, "Negative P-value. Something is wrong."
-            return dict(response=response, 
-                        pvalue=pval, 
-                        length=length,
-                        comment=comment)
-        else:
-            comment = "ignored (below minimal length)"
-            return dict(response=response, 
-                        pvalue=np.nan, 
-                        length=length,
-                        comment=comment)
-
-    def _get_pvals(self, responses: list, lengths: list) -> tuple:
-        pvals = []
-        comments = []
-        for response, length in zip(responses, lengths):
-            r = self._test_response(response, length)
-            pvals.append(float(r['pvalue']))
-            comments.append(r['comment'])
-        return pvals, comments
-
-
-    def _get_responses(self, sentences: list, contexts: list) -> list:
-        """
-        Compute response and length of a text sentence 
-        """
-        assert len(sentences) == len(contexts)
-
-        responses = []
-        lengths = []
-        for sent, ctx in tqdm(zip(sentences, contexts)):
-            logging.debug(f"Testing sentence: {sent} | context: {ctx}")
-            length = self._get_length(sent)
-            if self.length_limit_policy == 'truncate':
-                sent = truncae_to_max_no_tokens(sent, self.max_len)
-            if length == 1:
-                logging.warning(f"Sentence {sent} is too short. Skipping.")
-                responses.append(np.nan)
-                continue
-            try:
-                responses.append(self._test_sentence(sent, ctx))
-            except:
-                # something unusual happened...
-                import pdb; pdb.set_trace()
-            lengths.append(length)
-        return responses, lengths
-
-    def get_pvals(self, sentences: list, contexts: list) -> tuple:
-        """
-        logloss test of every (sentence, context) pair
-        """
-        assert len(sentences) == len(contexts)
-
-        responses, lengths = self._get_responses(sentences, contexts)
-        pvals, comments = self._get_pvals(responses, lengths)
-        
-        return pvals, responses, comments
-
-
-    def testHC(self, sentences: list) -> float:
-        pvals = np.array(self.get_pvals(sentences)[1])
-        mt = MultiTest(pvals, stbl=self.HC_stbl)
-        return mt.hc(gamma=0.4)[0]
-
-    def testFisher(self, sentences: list) -> dict:
-        pvals = np.array(self.get_pvals(sentences)[1])
-        print(pvals)
-        mt = MultiTest(pvals, stbl=self.HC_stbl)
-        return dict(zip(['Fn', 'pvalue'], mt.fisher()))
-
-    def _test_chunked_doc(self, lo_chunks: list, lo_contexts: list) -> tuple:
-        pvals, responses, comments = self.get_pvals(lo_chunks, lo_contexts)
-        if self.ignore_first_sentence:
-            pvals[0] = np.nan
-            logging.info('Ignoring the first sentence.')
-            comments[0] = "ignored (first sentence)"
-        
-        df = pd.DataFrame({'sentence': lo_chunks, 'response': responses, 'pvalue': pvals,
-                           'context': lo_contexts, 'comment': comments},
-                          index=range(len(lo_chunks)))
-        df_test = df[~df.pvalue.isna()]
-        if df_test.empty:
-            logging.warning('No valid chunks to test.')
-            return None, df
-        return MultiTest(df_test.pvalue, stbl=self.HC_stbl), df
-
-    def test_chunked_doc(self, lo_chunks: list, lo_contexts: list, dashboard=False) -> dict:
-        mt, df = self._test_chunked_doc(lo_chunks, lo_contexts)
-        if mt is None:
-            hc = np.nan
-            fisher = (np.nan, np.nan)
-            df['mask'] = pd.NA
-        else:
-            hc, hct = mt.hc(gamma=0.4)
-            fisher = mt.fisher()
-            df['mask'] = df['pvalue'] <= hct
-        if dashboard:
-            mt.hc_dashboard(gamma=0.4)
-        return dict(sentences=df, HC=hc, fisher=fisher[0], fisher_pvalue=fisher[1])
-
-    def __call__(self, lo_chunks: list, lo_contexts: list, dashboard=False) -> dict:
-        return self.test_chunked_doc(lo_chunks, lo_contexts, dashboard=dashboard)

Detect_Edits_in_AI-Generated_Text/src/HC_survival_function.py

@@ -1,66 +0,0 @@
-"""
-This script computes the survival function of the HC statistic for a given sample size n.
-The survival function is computed using a simulation of the null distribution of the HC statistic.
-We use the simulation results to fit a bivariate function of the form Pr[HC >= x | n] = f(n, x).
-The simulation results are saved in a file named HC_null_sim_results.csv.
-use function get_HC_survival_function to load the bivariate function or simulate the distribution. 
-"""
-
-import numpy as np
-import pandas as pd
-from multitest import MultiTest
-from tqdm import tqdm
-from scipy.interpolate import RectBivariateSpline
-from src.fit_survival_function import fit_survival_func
-import logging
-
-HC_NULL_SIM_FILE = "HC_null_sim_results.csv"
-STBL = True
-NN = [25, 50, 75, 100, 125, 150, 200, 250, 300, 400, 500]  # values of n to simulate
-
-def get_HC_survival_function(HC_null_sim_file, log_space=True, nMonte=10000, STBL=True):
-
-    xx = {}
-    if HC_null_sim_file is None:            
-            logging.info("Simulated HC null values file was not provided.")
-            for n in tqdm(NN):
-                logging.info(f"Simulating HC null values for n={n}...")
-                yy = np.zeros(nMonte)
-                for j in range(nMonte):
-                    uu = np.random.rand(n)
-                    mt = MultiTest(uu, stbl=STBL)
-                    yy[j] = mt.hc()[0]
-                xx[n] = yy
-            nn = NN # Idan
-    else:
-        logging.info(f"Loading HC null values from {HC_null_sim_file}...")
-        df = pd.read_csv(HC_null_sim_file, index_col=0)
-        for n in df.index:
-            xx[n] = df.loc[n]
-        nn = df.index.tolist()
-
-    xx0 = np.linspace(-1, 10, 57)
-    zz = []
-    for n in nn:
-        univariate_survival_func = fit_survival_func(xx[n], log_space=log_space)
-        zz.append(univariate_survival_func(xx0))
-        
-    func_log = RectBivariateSpline(np.array(nn), xx0, np.vstack(zz))
-
-    if log_space:
-        def func(x, y):
-            return np.exp(-func_log(x,y))
-        return func
-    else:
-        return func_log
-    
-
-def main():
-    func = get_HC_survival_function(HC_null_sim_file=HC_NULL_SIM_FILE, STBL=STBL)
-    print("Pr[HC >= 3 |n=50] = ", func(50, 3)[0][0]) # 9.680113e-05
-    print("Pr[HC >= 3 |n=100] = ", func(100, 3)[0][0]) # 0.0002335
-    print("Pr[HC >= 3 |n=200] = ", func(200, 3)[0][0]) # 0.00103771
-    
-
-if __name__ == '__main__':
-    main()

Detect_Edits_in_AI-Generated_Text/src/PerplexityEvaluator.py

@@ -1,34 +0,0 @@
-import torch
-
-class PerplexityEvaluator(object):
-    def __init__(self, model, tokenizer, ignore_index=-1):
-        self.model = model
-        self.tokenizer = tokenizer
-        self.ignore_index = ignore_index
-
-    def __call__(self, text, context=None):
-        return self.log_perplexity(text, context)
-
-    def log_perplexity(self, text, context=None):
-        """
-        Evaluate log perplexity of text with respect to the language model
-        based on the context
-
-        :param text:
-        :param context:
-        :return:
-        """
-        device = self.model.device
-        text_ids = self.tokenizer(text, return_tensors='pt')
-        if context:
-            context_ids = self.tokenizer(context, return_tensors='pt')
-            input_ids = torch.concatenate([context_ids['input_ids'], text_ids['input_ids']], axis=1)
-            labels = torch.concatenate([torch.ones_like(context_ids['input_ids']) * self.ignore_index,
-                                        text_ids['input_ids']], axis=1)
-            print("Warning, need to remove context length when reporting lppx")
-        else:
-            input_ids = text_ids['input_ids']
-            labels = input_ids
-
-        loss = self.model(input_ids=input_ids.to(device), labels=labels.to(device)).loss
-        return loss.cpu().detach().numpy()

Detect_Edits_in_AI-Generated_Text/src/PrepareArticles.py

@@ -1,74 +0,0 @@
-import pandas as pd
-import numpy as np
-import json
-
-class PrepareArticles(object):
-    """
-    Parse preprocessed data from csv
-
-    This information is needed for evaluating log-perplexity of the text with respect to a language model
-    and later on to test the likelihood that the sentence was sampled from the model with the relevant context.
-    """
-    def __init__(self, article_obj, get_edits=False, min_tokens=10, max_tokens=100, max_sentences=None):
-        self.article_obj = article_obj
-        self.min_tokens = min_tokens
-        self.max_tokens = max_tokens
-        self.get_edits = get_edits
-        self.max_sentences = max_sentences
-
-    def __call__(self, combined=True):
-        return self.parse_dataset(combined)
-    
-    def parse_dataset(self, combined=True):
-
-        texts = []
-        lengths = []
-        contexts = []
-        tags = []
-        
-        current_texts = []
-        current_lengths = []
-        current_contexts = []
-        current_tags = []
-        exceeded_max_sentences = False
-        
-        for sub_title in self.article_obj['sub_titles']: # For each sub title
-            for sentence in sub_title['sentences']: # Go over each sentence
-                sentence_size = len(sentence['sentence'].split())
-                if sentence_size >= self.min_tokens and sentence_size <= self.max_tokens:
-                    current_texts.append(sentence['sentence'])
-                    current_lengths.append(len(sentence['sentence'].split())) # Number of tokens
-                    current_contexts.append(sentence['context'] if 'context' in sentence else None)
-                    current_tags.append('no edits')
-
-                # If get_edits and has edited sentence save it
-                if self.get_edits and 'alternative' in sentence and len(sentence['alternative'].split()) >= self.min_tokens and len(sentence['alternative'].split()) <= self.max_tokens:
-                    current_texts.append(sentence['alternative'])
-                    current_lengths.append(len(sentence['alternative'].split()))
-                    current_contexts.append(sentence['alternative_context'] if 'alternative_context' in sentence else None)
-                    current_tags.append('<edit>')
-                if self.max_sentences and len(current_texts) >= self.max_sentences:
-                    exceeded_max_sentences = True
-                    break
-                    # return {'text': np.array(texts, dtype=object), 'length': np.array(lengths, dtype=object), 'context': np.array(contexts, dtype=object), 'tag': np.array(tags, dtype=object),
-                    #             'number_in_par': np.arange(1,1+len(texts))}
-            if exceeded_max_sentences:
-                break
-        
-        # If exceede max sentences only if self.max_sentences is not None
-        if (self.max_sentences and exceeded_max_sentences) or (not self.max_sentences):
-            # If combined, combine the data
-            if combined:
-                texts = texts + current_texts
-                lengths = lengths + current_lengths
-                contexts = contexts + current_contexts
-                tags = tags + current_tags
-            else:
-                texts.append(np.array(current_texts))
-                lengths.append(np.array(current_lengths))
-                contexts.append(np.array(current_contexts))
-                tags.append(np.array(current_tags))
-            
-        return {'text': np.array(texts, dtype=object), 'length': np.array(lengths, dtype=object), 'context': np.array(contexts, dtype=object), 'tag': np.array(tags, dtype=object),
-                    'number_in_par': np.arange(1,1+len(texts))}
-

Detect_Edits_in_AI-Generated_Text/src/PrepareSentenceContext.py

@@ -1,158 +0,0 @@
-import logging
-import spacy
-import re
-import numpy as np
-from src.SentenceParser import SentenceParser
-
-class PrepareSentenceContext(object):
-    """
-    Parse text and extract length and context information
-
-    This information is needed for evaluating log-perplexity of the text with respect to a language model
-    and later on to test the likelihood that the sentence was sampled from the model with the relevant context.
-    """
-
-    def __init__(self, sentence_parser='spacy', context_policy=None, context=None):
-        if sentence_parser == 'spacy':
-            self.nlp = spacy.load("en_core_web_sm", disable=["tagger", "attribute_ruler", "lemmatizer", "ner"])
-        if sentence_parser == 'regex':
-            logging.warning("Regex-based parser is not good at breaking sentences like 'Dr. Stone', etc.")
-            self.nlp = SentenceParser()
-            
-        self.sentence_parser_name = sentence_parser
-
-        self.context_policy = context_policy
-        self.context = context
-
-    def __call__(self, text):
-        return self.parse_sentences(text)
-
-    def parse_sentences(self, text):
-        pattern_close = r"(.*?)</edit>"
-        pattern_open = r"<edit>(.*?)"
-        MIN_TOKEN_LEN = 3
-
-        texts = []
-        tags = []
-        lengths = []
-        contexts = []
-
-        def update_sent(sent_text, tag, sent_length):
-            texts.append(sent_text)
-            tags.append(tag)
-            lengths.append(sent_length)
-            if self.context is not None:
-                context = self.context
-            elif self.context_policy is None:
-                context = None
-            elif self.context_policy == 'previous_sentence' and len(texts) > 0:
-                context = texts[-1]
-            else:
-                context = None
-            contexts.append(context)
-
-        curr_tag = None
-        parsed = self.nlp(text)
-        for s in parsed.sents:
-            prev_tag = curr_tag
-            matches_close = re.findall(pattern_close, s.text)
-            matches_open = re.findall(pattern_open, s.text)
-            matches_between = re.findall(r"<edit>(.*?)</edit>", s.text)
-            
-            logging.debug(f"Current sentence: {s.text}")
-            logging.debug(f"Matches open: {matches_open}")
-            logging.debug(f"Matches close: {matches_close}")
-            logging.debug(f"Matches between: {matches_between}")
-            if len(matches_close)>0 and len(matches_open)>0: 
-                logging.debug("Found an opening and a closing tag in the same sentence.")
-                if prev_tag is None and len(matches_open[0]) >= MIN_TOKEN_LEN:
-                    logging.debug("Openning followed by closing with some text in between.")
-                    update_sent(matches_open[0], "<edit>", len(s)-2)
-                    curr_tag = None
-                if prev_tag == "<edit>" and len(matches_close[0]) >= MIN_TOKEN_LEN:
-                    logging.warning(f"Wierd case: closing/openning followed by openning in sentence {len(texts)}")
-                    update_sent(matches_close[0], prev_tag, len(s)-1)
-                    curr_tag = None
-                if prev_tag == "</edit>":
-                    logging.debug("Closing followed by openning.")
-                    curr_tag = "<edit>"
-                    if len(matches_between[0]) > MIN_TOKEN_LEN:
-                        update_sent(matches_between[0], None, len(s)-2)
-            elif len(matches_open) > 0:
-                curr_tag = "<edit>"
-                assert prev_tag is None, f"Found an opening tag without a closing tag in sentence num. {len(texts)}"
-                if len(matches_open[0]) >= MIN_TOKEN_LEN:
-                    # text and tag are in the same sentence
-                    sent_text = matches_open[0]
-                    update_sent(sent_text, curr_tag, len(s)-1)      
-            elif len(matches_close) > 0:
-                curr_tag = "</edit>"
-                assert prev_tag == "<edit>", f"Found a closing tag without an opening tag in sentence num. {len(texts)}"
-                if len(matches_close[0]) >= MIN_TOKEN_LEN:
-                    # text and tag are in the same sentence
-                    update_sent(matches_close[0], prev_tag, len(s)-1)
-                curr_tag = None
-            else:
-                #if len(matches_close)==0 and len(matches_open)==0: 
-                # no tag
-                update_sent(s.text, curr_tag, len(s))
-        return {'text': texts, 'length': lengths, 'context': contexts, 'tag': tags,
-                    'number_in_par': np.arange(1,1+len(texts))}
-
-    def REMOVE_parse_sentences(self, text):
-        texts = []
-        contexts = []
-        lengths = []
-        tags = []
-        num_in_par = []
-        previous = None
-
-        text = re.sub("(</?[a-zA-Z0-9 ]+>\.?)\s+", r"\1.\n", text)  # to make sure that tags are in separate sentences
-        #text = re.sub("(</[a-zA-Z0-9 ]+>\.?)\s+", r"\n\1.\n", text)  # to make sure that tags are in separate sentences
-
-        parsed = self.nlp(text)
-
-        running_sent_num = 0
-        curr_tag = None
-        for i, sent in enumerate(parsed.sents):
-            # Here we try to track HTML-like tags. There might be
-            # some issues because spacy sentence parser has unexpected behavior when it comes to newlines
-            all_tags = re.findall(r"(</?[a-zA-Z0-9 ]+>)", str(sent))
-            if len(all_tags) > 1:
-                    logging.error(f"More than one tag in sentence {i}: {all_tags}")
-                    exit(1)
-            if len(all_tags) == 1:
-                tag = all_tags[0]
-                if tag[:2] == '</': # a closing tag
-                    if curr_tag is None:
-                        logging.warning(f"Closing tag without an opening tag in sentence {i}: {sent}")
-                    else:
-                        curr_tag = None
-                else:
-                    if curr_tag is not None:
-                        logging.warning(f"Opening tag without a closing tag in sentence {i}: {sent}")
-                    else:
-                        curr_tag = tag
-            else:  # if text is not a tag
-                sent_text = str(sent)
-                sent_length = len(sent)
-
-                texts.append(sent_text)
-                running_sent_num += 1
-                num_in_par.append(running_sent_num)
-                tags.append(curr_tag)
-                lengths.append(sent_length)
-
-                if self.context is not None:
-                    context = self.context
-                elif self.context_policy is None:
-                    context = None
-                elif self.context_policy == 'previous_sentence':
-                    context = previous
-                    previous = sent_text
-                else:
-                    context = None
-
-                contexts.append(context)
-        return {'text': texts, 'length': lengths, 'context': contexts, 'tag': tags,
-                'number_in_par': num_in_par}

Detect_Edits_in_AI-Generated_Text/src/SentenceParser.py

@@ -1,31 +0,0 @@
-import re
-
-
-class Sentence(object):
-    def __init__(self, text):
-        self.text = text
-        self.tokens = text.split()
-
-    def __len__(self):
-        return len(self.tokens)
-
-class Sentences(object):
-    def __init__(self, text):
-        def iterate(text):
-            for s in re.split(r"\n", text):
-                yield s
-        self.sents = iterate(text)
-
-    def __len__(self):
-        return len(self.sents)
-
-class SentenceParser(object):
-    """
-    Iterate over the text column of a dataframe
-    """
-
-    def __init__(self):
-        self.sents = None
-
-    def __call__(self, text):
-        return Sentences(text)

Detect_Edits_in_AI-Generated_Text/src/dataset_loaders.py

@@ -1,87 +0,0 @@
-from datasets import load_dataset
-
-SEED = 42
-
-
-def get_dataset(name: str, machine_field, human_field, iterable=False,
-                text_field=None, shuffle=False, main_split='train'):
-    dataset = load_dataset(name)[main_split]
-    ds = dataset.rename_columns({human_field: 'human_text', machine_field: 'machine_text'})
-    if 'id' not in ds.features:
-        ids = list(range(len(ds)))
-        ds = ds.add_column("id", ids)
-    if text_field:
-        ds = ds.rename_columns({text_field: 'text'})
-
-    if iterable:
-        ds = ds.to_iterable_dataset()
-    if shuffle:
-        return ds.shuffle(seed=SEED)
-    else:
-        return ds
-
-
-def get_text_from_wiki_dataset(shuffle=False, text_field=None):
-    return get_dataset(name="aadityaubhat/GPT-wiki-intro", machine_field='generated_intro',
-                       human_field="wiki_intro", shuffle=shuffle, text_field=text_field)
-
-
-def get_text_from_wiki_long_dataset(shuffle=False, text_field=None):
-    return get_dataset(name="alonkipnis/wiki-intro-long", machine_field='generated_intro',
-                       human_field="wiki_intro", shuffle=shuffle, text_field=text_field)
-
-
-def get_text_from_wiki_long_dataset_local(shuffle=False, text_field=None, iterable=False):
-    """
-    A version of wiki_intro dataset with at least 15 sentences per generated article
-    """
-    dataset = load_dataset("alonkipnis/wiki-intro-long")
-    ds = dataset.rename_columns({"wiki_intro": 'human_text', "generated_intro": 'machine_text'})
-    if text_field:
-        ds = ds.rename_columns({text_field: 'text'})
-    if iterable:
-        ds = ds.to_iterable_dataset()
-    if shuffle:
-        return ds.shuffle(seed=SEED)
-    else:
-        return ds
-
-
-def get_text_from_chatgpt_news_long_dataset_local(shuffle=False, text_field=None, iterable=False):
-    """
-    A version of chatgpt-news-articles dataset with at least 15 sentences per generated article
-    Only 'train' split is included
-    """
-    dataset = load_dataset("alonkipnis/news-chatgpt-long")
-    ds = dataset.rename_columns({"article": 'human_text', "chatgpt": 'machine_text'})
-    if text_field:
-        ds = ds.rename_columns({text_field: 'text'})
-    if iterable:
-        ds = ds.to_iterable_dataset()
-    if shuffle:
-        return ds.shuffle(seed=SEED)
-    else:
-        return ds
-
-def get_text_from_chatgpt_abstracts_dataset(shuffle=False, text_field=None):
-    return get_dataset(name="NicolaiSivesind/ChatGPT-Research-Abstracts", machine_field="generated_abstract",
-                       human_field="real_abstract", shuffle=shuffle, text_field=text_field)
-
-def get_text_from_chatgpt_news_long_dataset(shuffle=False, text_field=None):
-    return get_dataset(name="alonkipnis/news-chatgpt-long", machine_field='chatgpt',
-                       human_field="article", shuffle=shuffle, text_field=text_field)
-
-
-def get_text_from_chatgpt_news_dataset(shuffle=False, text_field=None):
-    return get_dataset(name="isarth/chatgpt-news-articles", machine_field='chatgpt',
-                       human_field="article", shuffle=shuffle, text_field=text_field)
-
-
-def get_text_from_wikibio_dataset(shuffle=False, text_field=None):
-    return get_dataset(name="potsawee/wiki_bio_gpt3_hallucination", machine_field='gpt3_text',
-                       human_field="wiki_bio_text", shuffle=shuffle, text_field=text_field, main_split='evaluation')
-
-## New datasets (22/5/2023)
-def get_text_from_alpaca_gpt4_dataset(shuffle=False, text_field=None):
-    return get_dataset(name="polyware-ai/alpaca-gpt4-cleaned", machine_field='output',
-                       human_field="instruction", shuffle=shuffle, text_field=text_field)

Detect_Edits_in_AI-Generated_Text/src/fit_survival_function.py

@@ -1,94 +0,0 @@
-"""
-Script to read log-loss data of many sentences and characterize the empirical distribution.
-We also report the mean log-loss as a function of sentence length
-"""
-from scipy.interpolate import RectBivariateSpline, interp1d
-import numpy as np
-
-def fit_survival_func(xx, log_space=True):
-    """
-    Returns an estimated survival function to the data in :xx: using
-    interpolation.
-
-    Args:
-        :xx:  data
-        :log_space:  indicates whether fitting is in log space or not.
-
-    Returns:
-         univariate function
-    """
-    assert len(xx) > 0
-
-    eps = 1 / len(xx)
-    inf = 1 / eps
-
-    sxx = np.sort(xx)
-    qq = np.mean(np.expand_dims(sxx,1) >= sxx, 0)
-
-    if log_space:
-        qq = -np.log(qq)
-
-
-    if log_space:
-        return interp1d(sxx, qq, fill_value=(0 , np.log(inf)), bounds_error=False)
-    else:
-        return interp1d(sxx, qq, fill_value=(1 , 0), bounds_error=False)
-
-
-def fit_per_length_survival_function(lengths, xx, G=501, log_space=True):
-    """
-    Returns a survival function for every sentence length in tokens.
-    Use 2D interpolation over the empirical survival function of the pairs (length, x)
-    
-    Args:
-        :lengths:, :xx:, 1-D arrays
-        :G:  number of grid points to use in the interpolation in the xx dimension
-        :log_space:  indicates whether result is in log space or not.
-
-    Returns:
-        bivariate function (length, x) -> [0,1]
-    """
-
-    assert len(lengths) == len(xx)
-
-    min_tokens_per_sentence = lengths.min()
-    max_tokens_per_sentence = lengths.max()
-    ll = np.arange(min_tokens_per_sentence, max_tokens_per_sentence)
-
-    ppx_min_val = xx.min()
-    ppx_max_val = xx.max()
-    xx0 = np.linspace(ppx_min_val, ppx_max_val, G)
-
-    ll_valid = []
-    zz = []
-    for l in ll:
-        xx1 = xx[lengths == l]
-        if len(xx1) > 1:
-            univariate_survival_func = fit_survival_func(xx1, log_space=log_space)
-            ll_valid.append(l)
-            zz.append(univariate_survival_func(xx0))
-
-    func = RectBivariateSpline(np.array(ll_valid), xx0, np.vstack(zz))
-    if log_space:
-        def func2d(x, y):
-            return np.exp(-func(x,y))
-        return func2d
-    else:
-        return func
-    
-
-# import pickle
-# import pandas as pd
-# df = pd.read_csv('D:\\.Idan\\תואר שני\\תזה\\detectLM\\article_null.csv')
-# LOGLOSS_PVAL_FUNC_FILE = 'D:\.Idan\תואר שני\תזה\detectLM\example\logloss_pval_function.pkl'
-# LOGLOSS_PVAL_FUNC_FILE_TEST = 'D:\.Idan\תואר שני\תזה\detectLM\example\logloss_pval_function_test.pkl'
-# with open(LOGLOSS_PVAL_FUNC_FILE, 'wb') as handle:
-#     pickle.dump(fit_per_length_survival_function(df['length'].values, df['response'].values), handle, protocol=pickle.HIGHEST_PROTOCOL)
-
-# with open(LOGLOSS_PVAL_FUNC_FILE, 'rb') as f:
-#     data = pickle.load(f)
-#     print(data)
-
-# with open(LOGLOSS_PVAL_FUNC_FILE_TEST, 'rb') as f:
-#     data = pickle.load(f)
-#     print(data)

Detect_Edits_in_AI-Generated_Text/threshold_obj.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5b9b1dfc4fc552a4c975ebe1f05a5140bee30fc8231fd4b4eba1dcf4082d127a
-size 208