ultima versao

app.py

@@ -4,6 +4,13 @@ from textblob import TextBlob
 import textstat
 from huggingface_hub import hf_hub_download
 from joblib import load
+from util import escape_tags_and_content, escape_tags, escape_strings, escape_links, escape_hex_character_codes, escape_punctuation_boundaries, escape_odd_spaces
+
+import nltk
+import nltk
+from nltk.corpus import stopwords
+ 
+nltk.download('stopwords')
 
 titulo1 = """CLONE - Studio Dashboard: "default" and "Default Project" does not give clear information about Alloy and Project unless description is read."""
 descricao1 = """Steps To Reproduce: 1. On dashboard on studio 3.0, navigate to Develop tab. 2. Notice "default" and "Default Project" & "two-tabbed" and "Tabbed Application" names. Actual: User does not get clear information from names that one is alloy project and another one is Titanium project unless he reads the description below. Expected: Naming convention or icon corresponding must suggest type"""
@@ -20,104 +27,144 @@ descricao4 = """During the compile process Alloy will attempt to remove files fr
 titulo5 = """Resolve suboptimal compression from uglify-js v2 update"""
 descricao5 = """The v2 update of uglify-js in Alloy, specifically version 2.2.5, has some suboptimal compressions, which are causing the optimizer.js test spec to fail in certain cases. Specifically the issues are around booleans and cascading of variables in assignments. These issues have been logged with the Uglifyjs2 project in the following links:    * https://github.com/mishoo/UglifyJS2/issues/137  * https://github.com/mishoo/UglifyJS2/issues/138    When these issues are resolved and distributed in an npm release, we need to revisit these compressions and testing to ensure that the fixes are in place, and that new uglify-js version has no regressions that impact alloy."""
 
-def calcula_MbR(titulo, descricao):
+def calcula_MbR(titulo, descricao, nome_projeto):
     context = titulo + descricao
-    d = {"context": [context]}
-    df = pd.DataFrame(data=d, columns=["context"])
-    model = load(hf_hub_download("model_effort_tawos", "model_tawos_aloy_mbr.joblib"))
-    story_points_MbR = model.predict(df["context"])
+    d = {"context_": [context]}
+    df = pd.DataFrame(data=d, columns=["context_"])
+    model = load(hf_hub_download("giseldo/model_effort_tawos", "model_tawos_aloy_mbr.joblib", force_download=False))
+    story_points_MbR = model.predict(df["context_"])
     return story_points_MbR
 
-def calcula_Median(titulo, descricao):
+def calcula_Median(titulo, descricao, nome_projeto):
     context = titulo + descricao
-    d = {"context": [context]}
-    df = pd.DataFrame(data=d, columns=["context"])
-    model = load(hf_hub_download("giseldo/model_effort_tawos", "model_tawos_aloy_median.joblib"))
-    story_points_MbR = model.predict(df["context"])
+    d = {"context_": [context]}
+    df = pd.DataFrame(data=d, columns=["context_"])
+    model = load(hf_hub_download("giseldo/model_effort_tawos", "model_tawos_aloy_median.joblib", force_download=False))
+    story_points_MbR = model.predict(df["context_"])
     return story_points_MbR
 
-def calcula_NEOSP_SVR(titulo, descricao):
-    model = load(hf_hub_download("giseldo/model_effort_tawos", "model_tawos_aloy_neosp_svr.joblib"))
+def calcula_NEOSP_SVR(titulo, descricao, nome_projeto):
+    model = load(hf_hub_download("giseldo/model_effort_tawos", "model_tawos_aloy_neosp_svr.joblib", force_download=False))
+    
+    # criação de uma nova coluna
     context = titulo + descricao
     d = {"context": [context]}
     df = pd.DataFrame(data=d, columns=["context"])
 
+    # pré-processamento
+    df["context"] = df["context"].apply(lambda x: escape_tags_and_content(x))
+    df["context"] = df["context"].apply(lambda x: escape_tags(x))
+    df["context"] = df["context"].apply(lambda x: escape_strings(x))
+    df["context"] = df["context"].apply(lambda x: escape_links(x))
+    df["context"] = df["context"].apply(lambda x: escape_hex_character_codes(x))
+    df["context"] = df["context"].apply(lambda x: escape_punctuation_boundaries(x))
+    df["context"] = df["context"].apply(lambda x: escape_odd_spaces(x))
+
+    # removendo stop-words
+    stop = stopwords.words('english')
+    df['context'] = df['context'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))
+
+    # renomeando as colunas porque senão dá um problema com a extração de features do NEOSP
+    df = df.rename(columns={ "context": "context_"})
+    
     # features de legibilidade
-    df["gunning_fog"] = df['context'].apply(textstat.gunning_fog)#
-    df["flesch_reading_ease"] = df['context'].apply(textstat.flesch_reading_ease)#
-    df["flesch_kincaid_grade"] = df['context'].apply(textstat.flesch_kincaid_grade)#
-    df["smog_index"] = df['context'].apply(textstat.smog_index)
-    df["coleman_liau_index"] = df['context'].apply(textstat.coleman_liau_index)#
-    df["automated_readability_index"] = df['context'].apply(textstat.automated_readability_index) #
-    df["dale_chall_readability_score"] = df['context'].apply(textstat.dale_chall_readability_score)#
-    df["difficult_words"] = df['context'].apply(textstat.difficult_words)
-    df["linsear_write_formula"] = df['context'].apply(textstat.linsear_write_formula)#
+    df["gunning_fog_"] = df['context_'].apply(textstat.gunning_fog)#
+    df["flesch_reading_ease_"] = df['context_'].apply(textstat.flesch_reading_ease)#
+    df["flesch_kincaid_grade_"] = df['context_'].apply(textstat.flesch_kincaid_grade)#
+    df["smog_index_"] = df['context_'].apply(textstat.smog_index)
+    df["coleman_liau_index_"] = df['context_'].apply(textstat.coleman_liau_index)#
+    df["automated_readability_index_"] = df['context_'].apply(textstat.automated_readability_index) #
+    df["dale_chall_readability_score_"] = df['context_'].apply(textstat.dale_chall_readability_score)#
+    df["difficult_words_"] = df['context_'].apply(textstat.difficult_words)
+    df["linsear_write_formula_"] = df['context_'].apply(textstat.linsear_write_formula)#
 
     # feature de sentimento
-    df["polarity"] = df["context"].apply(lambda x: TextBlob(x).sentiment.polarity)
-    df["subjectivity"] = df["context"].apply(lambda x: TextBlob(x).sentiment.subjectivity)
+    df["polarity_"] = df["context_"].apply(lambda x: TextBlob(x).sentiment.polarity)
+    df["subjectivity_"] = df["context_"].apply(lambda x: TextBlob(x).sentiment.subjectivity)
 
-    X = df[["gunning_fog", "flesch_reading_ease", "flesch_kincaid_grade", "smog_index", "coleman_liau_index", 
-            "automated_readability_index", "dale_chall_readability_score", "difficult_words", "linsear_write_formula", 
-            "polarity", "subjectivity"]]
+    X = df[["gunning_fog_", "flesch_reading_ease_", "flesch_kincaid_grade_", "smog_index_", "coleman_liau_index_", 
+            "automated_readability_index_", "dale_chall_readability_score_", "difficult_words_", "linsear_write_formula_", 
+            "polarity_", "subjectivity_"]]
 
     story_points = model.predict(X)
     return story_points
 
-def calcula_NEOSP_Linear(titulo, descricao):
-    model = load(hf_hub_download("giseldo/model_effort_tawos", "model_tawos_aloy_neosp_linear.joblib"))
+def calcula_NEOSP_Linear(titulo, descricao, nome_projeto):
+    model = load(hf_hub_download("giseldo/model_effort_tawos", "model_tawos_aloy_neosp_linear.joblib", force_download=False))
+    # criação de uma nova coluna
     context = titulo + descricao
     d = {"context": [context]}
     df = pd.DataFrame(data=d, columns=["context"])
 
+    # pré-processamento
+    df["context"] = df["context"].apply(lambda x: escape_tags_and_content(x))
+    df["context"] = df["context"].apply(lambda x: escape_tags(x))
+    df["context"] = df["context"].apply(lambda x: escape_strings(x))
+    df["context"] = df["context"].apply(lambda x: escape_links(x))
+    df["context"] = df["context"].apply(lambda x: escape_hex_character_codes(x))
+    df["context"] = df["context"].apply(lambda x: escape_punctuation_boundaries(x))
+    df["context"] = df["context"].apply(lambda x: escape_odd_spaces(x))
+
+    # removendo stop-words
+    stop = stopwords.words('english')
+    df['context'] = df['context'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))
+
+    # renomeando as colunas porque senão dá um problema com a extração de features do NEOSP
+    df = df.rename(columns={ "context": "context_"})
+
     # features de legibilidade
-    df["gunning_fog"] = df['context'].apply(textstat.gunning_fog)#
-    df["flesch_reading_ease"] = df['context'].apply(textstat.flesch_reading_ease)#
-    df["flesch_kincaid_grade"] = df['context'].apply(textstat.flesch_kincaid_grade)#
-    df["smog_index"] = df['context'].apply(textstat.smog_index)
-    df["coleman_liau_index"] = df['context'].apply(textstat.coleman_liau_index)#
-    df["automated_readability_index"] = df['context'].apply(textstat.automated_readability_index) #
-    df["dale_chall_readability_score"] = df['context'].apply(textstat.dale_chall_readability_score)#
-    df["difficult_words"] = df['context'].apply(textstat.difficult_words)
-    df["linsear_write_formula"] = df['context'].apply(textstat.linsear_write_formula)#
+    df["gunning_fog_"] = df['context_'].apply(textstat.gunning_fog)#
+    df["flesch_reading_ease_"] = df['context_'].apply(textstat.flesch_reading_ease)#
+    df["flesch_kincaid_grade_"] = df['context_'].apply(textstat.flesch_kincaid_grade)#
+    df["smog_index_"] = df['context_'].apply(textstat.smog_index)
+    df["coleman_liau_index_"] = df['context_'].apply(textstat.coleman_liau_index)#
+    df["automated_readability_index_"] = df['context_'].apply(textstat.automated_readability_index) #
+    df["dale_chall_readability_score_"] = df['context_'].apply(textstat.dale_chall_readability_score)#
+    df["difficult_words_"] = df['context_'].apply(textstat.difficult_words)
+    df["linsear_write_formula_"] = df['context_'].apply(textstat.linsear_write_formula)#
 
     # feature de sentimento
-    df["polarity"] = df["context"].apply(lambda x: TextBlob(x).sentiment.polarity)
-    df["subjectivity"] = df["context"].apply(lambda x: TextBlob(x).sentiment.subjectivity)
+    df["polarity_"] = df["context_"].apply(lambda x: TextBlob(x).sentiment.polarity)
+    df["subjectivity_"] = df["context_"].apply(lambda x: TextBlob(x).sentiment.subjectivity)
 
-    X = df[["gunning_fog", "flesch_reading_ease", "flesch_kincaid_grade", "smog_index", "coleman_liau_index", 
-            "automated_readability_index", "dale_chall_readability_score", "difficult_words", "linsear_write_formula", 
-            "polarity", "subjectivity"]]
+    X = df[["gunning_fog_", "flesch_reading_ease_", "flesch_kincaid_grade_", "smog_index_", "coleman_liau_index_", 
+            "automated_readability_index_", "dale_chall_readability_score_", "difficult_words_", "linsear_write_formula_", 
+            "polarity_", "subjectivity_"]]
 
     story_points = model.predict(X)
     return story_points
     
-def calcula_TFIDF_SVR(titulo, descricao):
-    model = load(hf_hub_download("giseldo/model_effort_tawos", "model_tawos_aloy_tfidf_svr.joblib"))
+def calcula_TFIDF_SVR(titulo, descricao, nome_projeto):
+    model = load(hf_hub_download("giseldo/model_effort_tawos", "model_tawos_aloy_tfidf_svr.joblib", force_download=False))
     context = titulo + descricao
-    d = {"context": [context]}
-    df = pd.DataFrame(data=d, columns=["context"])
-    vectorizer = load(hf_hub_download("giseldo/model_effort_tawos", "vectorizer_tfidf.joblib"))
-    X = vectorizer.transform(df["context"])
+    d = {"context_": [context]}
+    df = pd.DataFrame(data=d, columns=["context_"])
+    vectorizer = load(hf_hub_download("giseldo/model_effort_tawos", "vectorizer_tfidf.joblib", force_download=False))
+    X_vec = vectorizer.transform(df["context_"])
+    df_vec = pd.DataFrame(data = X_vec.toarray(), columns = vectorizer.get_feature_names_out())
+    X = df_vec
     story_points = model.predict(X)
     return story_points
 
-def calcula_TFIDF_Linear(titulo, descricao):
-    model = load(hf_hub_download("giseldo/model_effort_tawos", "model_tawos_aloy_tfidf_linear.joblib"))
+def calcula_TFIDF_Linear(titulo, descricao, nome_projeto):
+    model = load(hf_hub_download("giseldo/model_effort_tawos", "model_tawos_aloy_tfidf_linear.joblib", force_download=False))
     context = titulo + descricao
-    d = {"context": [context]}
-    df = pd.DataFrame(data=d, columns=["context"])
-    vectorizer = load(hf_hub_download("giseldo/model_effort_tawos", "vectorizer_tfidf.joblib"))
-    X = vectorizer.transform(df["context"])
+    d = {"context_": [context]}
+    df = pd.DataFrame(data=d, columns=["context_"])
+    vectorizer = load(hf_hub_download("giseldo/model_effort_tawos", "vectorizer_tfidf.joblib", force_download=False))
+    X_vec = vectorizer.transform(df["context_"])
+    df_vec = pd.DataFrame(data = X_vec.toarray(), columns = vectorizer.get_feature_names_out())
+    X = df_vec
     story_points = model.predict(X)
     return story_points
 
-def calcula(titulo, descricao):
-    return calcula_MbR(titulo, descricao), calcula_Median(titulo, descricao), calcula_NEOSP_SVR(titulo, descricao), calcula_NEOSP_Linear(titulo, descricao), calcula_TFIDF_SVR(titulo, descricao), calcula_TFIDF_Linear(titulo, descricao)
+def calcula(titulo, descricao, nome_projeto):
+    return calcula_MbR(titulo, descricao, nome_projeto), calcula_Median(titulo, descricao, nome_projeto), calcula_NEOSP_SVR(titulo, descricao, nome_projeto), calcula_NEOSP_Linear(titulo, descricao, nome_projeto),  calcula_TFIDF_SVR(titulo, descricao, nome_projeto),  calcula_TFIDF_Linear(titulo, descricao, nome_projeto)
 
 demo = gr.Interface(fn=calcula, 
                     inputs=[gr.Textbox(placeholder="Título", label="Título"), 
-                            gr.Textbox(lines=10, placeholder="Descrição", label="Descrição")], 
+                            gr.Textbox(lines=10, placeholder="Descrição", label="Descrição"),
+                            gr.Dropdown(["ALOY", "XD", "TIMOB"], label="Projeto", value= "ALOY", interactive= False)], # info="Nome do projeto!"
                     outputs=[gr.Textbox(label="Story Points Estimado Média"),
                              gr.Textbox(label="Story Points Estimado Mediana"),
                              gr.Textbox(label="Story Points Estimado NEOSP-SVR"),
@@ -125,6 +172,7 @@ demo = gr.Interface(fn=calcula,
                              gr.Textbox(label="Story Points Estimado TFIDF-SVR"),
                              gr.Textbox(label="Story Points Estimado TFIDF-Linear")],
                     title="Agile Task Story Point Estimator",
+                    #interpretation="default",
                     examples=[[titulo1, descricao1], [titulo2, descricao2], [titulo3, descricao3], [titulo4, descricao4], [titulo5, descricao5]])
 
 demo.launch() 

flagged/log.csv

@@ -0,0 +1,2 @@
+Título,Descrição,Projeto,Story Points Estimado Média,Story Points Estimado Mediana,Story Points Estimado NEOSP-SVR,Story Points Estimado NEOSP-Linear,Story Points Estimado TFIDF-SVR,Story Points Estimado TFIDF-Linear,flag,username,timestamp
+"CLONE - Studio Dashboard: ""default"" and ""Default Project"" does not give clear information about Alloy and Project unless description is read.","Steps To Reproduce: 1. On dashboard on studio 3.0, navigate to Develop tab. 2. Notice ""default"" and ""Default Project"" & ""two-tabbed"" and ""Tabbed Application"" names. Actual: User does not get clear information from names that one is alloy project and another one is Titanium project unless he reads the description below. Expected: Naming convention or icon corresponding must suggest type",ALOY,[3.70539419],[3.],[3.15496615],[3.84946518],[3.75963544],[6.99007244],,,2023-08-31 23:02:15.199700

requirements.txt

@@ -2,3 +2,4 @@ scikit-learn
 gradio
 textblob
 textstat
+nltk

util.py

@@ -0,0 +1,60 @@
+import re
+from string import punctuation
+
+def escape_tags_and_content(text):
+    """Escape tags and their content containing text, which is not written in natural language, such as code snippets"""
+
+    NO_TEXT_TAGS = "code", "noformat"
+    for tag in NO_TEXT_TAGS:
+        regex_matching_tag = re.compile("\{%s(.*?)\}(.*?)\{%s\}" % (tag, tag), re.DOTALL)
+        text = re.sub(regex_matching_tag, "", text)
+
+    return text
+
+def escape_tags(text):
+    """Escape markup tags, but retain their content"""
+
+    ESCAPE_TAGS = "color", "quote", "anchor", "panel"
+    for tag in  ESCAPE_TAGS:
+        text = re.sub("\{%s(.*?)\}" % tag, "", text)
+
+    return text
+
+def escape_strings(text):
+    """Escape line breaks, tabulators, slashes and JIRA heading markup symbols"""
+
+    ESCAPE_STRINGS = "\\r", "\\n", "\\t", "\\f", "\\v", "\"", "\\\\", "h1. ", "h2. ", "h3. ", "h4. ", "h5. ", "h6. "
+    for escape_string in ESCAPE_STRINGS:
+        text = text.replace(escape_string, " ")
+
+    return text
+
+def escape_links(text):
+    """Escape external and internal links, recognized by JIRA markup or leading 'http://' or 'https://' """
+
+    LINK_STARTERS = r"\#", r"\^", r"http\:\/\/", r"https\:\/\/", r"malto\:", r"file\:", r"\~"
+    for link_starter in LINK_STARTERS:
+        text = re.sub("\[(.*?\\|)?%s(.*?)\]" % link_starter, "", text)
+        text = re.sub(r"\bhttps?://\S+", "", text)
+
+    return text
+
+def escape_hex_character_codes(text):
+    """Escape characters outside the latin alphabet which are converted to hex code representation"""
+
+    return re.sub(r"\\x\w\w", "", text)
+
+def escape_punctuation_boundaries(text):
+    """Remove all punctuation marks from the beginning and end of words,
+    except for trailing period at the end of words"""
+
+    return " ".join([word.strip(punctuation.replace(".", "")).lstrip(".") for word in text.split()])
+
+def escape_odd_spaces(text):
+    """Replace several consequent spaces with one space
+    and remove spaces from string start and end"""
+
+    text = re.sub(r"\s+", " ", text)
+    text = text.strip()
+
+    return text