Spaces:
Running
Running
File size: 4,388 Bytes
a84c0b0 0920957 a84c0b0 0920957 a84c0b0 0920957 a84c0b0 0920957 a84c0b0 0920957 a84c0b0 0920957 a84c0b0 0920957 a84c0b0 0920957 a84c0b0 0920957 a84c0b0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 |
import streamlit as st
import requests
from bs4 import BeautifulSoup
import pandas as pd
from transformers import pipeline, AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments
from datasets import load_dataset, Dataset
# OSINT functions
def get_github_stars_forks(owner, repo):
url = f"https://api.github.com/repos/{owner}/{repo}"
response = requests.get(url)
data = response.json()
return data['stargazers_count'], data['forks_count']
def get_github_issues(owner, repo):
url = f"https://api.github.com/repos/{owner}/{repo}/issues"
response = requests.get(url)
issues = response.json()
return len(issues)
def get_github_pull_requests(owner, repo):
url = f"https://api.github.com/repos/{owner}/{repo}/pulls"
response = requests.get(url)
pulls = response.json()
return len(pulls)
def get_github_license(owner, repo):
url = f"https://api.github.com/repos/{owner}/{repo}/license"
response = requests.get(url)
data = response.json()
return data['license']['name']
def get_last_commit(owner, repo):
url = f"https://api.github.com/repos/{owner}/{repo}/commits"
response = requests.get(url)
commits = response.json()
return commits[0]['commit']['committer']['date']
def get_github_workflow_status(owner, repo):
url = f"https://api.github.com/repos/{owner}/{repo}/actions/runs"
response = requests.get(url)
runs = response.json()
return runs['workflow_runs'][0]['status'] if runs['workflow_runs'] else "No workflows found"
# Function to fetch page title from a URL
def fetch_page_title(url):
try:
response = requests.get(url)
st.write(f"Fetching URL: {url} - Status Code: {response.status_code}")
if response.status_code == 200:
soup = BeautifulSoup(response.text, 'html.parser')
title = soup.title.string if soup.title else 'No title found'
return title
else:
return f"Error: Received status code {response.status_code}"
except Exception as e:
return f"An error occurred: {e}"
# Main Streamlit app
def main():
st.title("OSINT Tool")
st.write("### GitHub Repository OSINT Analysis")
st.write("Enter the GitHub repository owner and name:")
owner = st.text_input("Repository Owner")
repo = st.text_input("Repository Name")
if owner and repo:
stars, forks = get_github_stars_forks(owner, repo)
open_issues = get_github_issues(owner, repo)
open_pulls = get_github_pull_requests(owner, repo)
license_type = get_github_license(owner, repo)
last_commit = get_last_commit(owner, repo)
workflow_status = get_github_workflow_status(owner, repo)
st.write(f"Stars: {stars}, Forks: {forks}")
st.write(f"Open Issues: {open_issues}, Open Pull Requests: {open_pulls}")
st.write(f"License: {license_type}")
st.write(f"Last Commit: {last_commit}")
st.write(f"Workflow Status: {workflow_status}")
st.write("### URL Title Fetcher")
url = st.text_input("Enter a URL to fetch its title:")
if url:
title = fetch_page_title(url)
st.write(f"Title: {title}")
st.write("### Dataset Upload & Model Fine-Tuning")
dataset_file = st.file_uploader("Upload a CSV file for fine-tuning", type=["csv"])
if dataset_file:
df = pd.read_csv(dataset_file)
st.dataframe(df.head())
st.write("Select a model for fine-tuning:")
model_name = st.selectbox("Model", ["bert-base-uncased", "distilbert-base-uncased"])
if st.button("Fine-tune Model"):
if dataset_file:
dataset = Dataset.from_pandas(df)
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name)
def tokenize_function(examples):
return tokenizer(examples['text'], padding="max_length", truncation=True)
tokenized_datasets = dataset.map(tokenize_function, batched=True)
training_args = TrainingArguments(output_dir="./results", num_train_epochs=1, per_device_train_batch_size=8)
trainer = Trainer(model=model, args=training_args, train_dataset=tokenized_datasets)
trainer.train()
st.write("Model fine-tuned successfully!")
if __name__ == "__main__":
main()
|