Update src/DetectLM.py

src/DetectLM.py

@@ -11,7 +11,7 @@ def truncae_to_max_no_tokens(text, max_no_tokens):
 
 class DetectLM(object):
     def __init__(self, sentence_detection_function, survival_function_per_length,
-                 min_len=4, max_len=100, HC_type="stbl",
+                 min_len=4, max_len=100, HC_type="stbl", gamma=0.15,
                  length_limit_policy='truncate', ignore_first_sentence=False):
         """
         Test for the presence of sentences of irregular origin as reflected by the
@@ -41,6 +41,7 @@ class DetectLM(object):
         self.length_limit_policy = length_limit_policy
         self.ignore_first_sentence = ignore_first_sentence
         self.HC_stbl = True if HC_type == 'stbl' else False
+        self.gamma = gamma
 
     def _logperp(self, sent: str, context=None) -> float:
         return float(self.sentence_detector(sent, context))
@@ -136,7 +137,7 @@ class DetectLM(object):
     def testHC(self, sentences: list) -> float:
         pvals = np.array(self.get_pvals(sentences)[1])
         mt = MultiTest(pvals, stbl=self.HC_stbl)
-        return mt.hc(gamma=0.4)[0]
+        return mt.hc(gamma=self.gamma)[0]
 
     def testFisher(self, sentences: list) -> dict:
         pvals = np.array(self.get_pvals(sentences)[1])
@@ -167,11 +168,11 @@ class DetectLM(object):
             fisher = (np.nan, np.nan)
             df['mask'] = pd.NA
         else:
-            hc, hct = mt.hc(gamma=0.4)
+            hc, hct = mt.hc(gamma=self.gamma)
             fisher = mt.fisher()
             df['mask'] = df['pvalue'] <= hct
         if dashboard:
-            mt.hc_dashboard(gamma=0.4)
+            mt.hc_dashboard(gamma=self.gamma)
         return dict(sentences=df, HC=hc, fisher=fisher[0], fisher_pvalue=fisher[1])
 
     def __call__(self, lo_chunks: list, lo_contexts: list, dashboard=False) -> dict: