File size: 3,856 Bytes
c9b0b96 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 |
#!/usr/bin/env python3
import requests
import json
from collections import Counter
def get_model_counts():
# Base URL of the API endpoint
url = "https://civitai.com/api/v1/models"
# Parameters for the API request
params = {
'limit': 100, # Set a high limit to reduce the number of pages to iterate through
'page': 1 # Start from the first page
}
# Initialize counts
total_models = 0
# counter
allow_commercial_use = []
creator = []
# sum
nsfw_count = []
allow_derivatives = []
allow_no_credit = []
download_count = []
favorite_count = []
comment_count = []
rating_count = []
tipped_amount_count = []
ratings = []
failures = 0
total_pages = 1000 # set to very high value, just in case it fails a lot in the beginning
while True:
# Make the GET request
try:
response = requests.get(url, params=params)
response.raise_for_status() # This will raise an error if the request fails
data = response.json()
# Process the current page of results
models = data["items"]
total_models += len(models)
for model in models:
allow_commercial_use.append(model["allowCommercialUse"])
creator.append(model["creator"]["username"])
nsfw_count.append(model["nsfw"])
allow_derivatives.append(model["allowDerivatives"])
allow_no_credit.append(model["allowNoCredit"])
download_count.append(model["stats"]["downloadCount"])
favorite_count.append(model["stats"]["favoriteCount"])
comment_count.append(model["stats"]["commentCount"])
rating_count.append(model["stats"]["ratingCount"])
tipped_amount_count.append(model["stats"]["tippedAmountCount"])
ratings.append(model["stats"]["rating"])
# Check if there are more pages to process
total_pages = int(data.get('metadata', {}).get('totalPages', 0))
# if params['page'] >= total_pages:
if params['page'] >= 2:
break
# Increment the page number for the next request
params['page'] += 1
print(f"{params['page']} / {total_pages}")
except:
failures += 1
params["page"] += 1
return {
'total_models': total_models,
'allow_commercial_use': allow_commercial_use,
'creator': creator,
'nsfw_count': nsfw_count,
'allow_derivatives': allow_derivatives,
'allow_no_credit': allow_no_credit,
'download_count': download_count,
'favorite_count': favorite_count,
'comment_count': comment_count,
'rating_count': rating_count,
'tipped_amount_count': tipped_amount_count,
'ratings': ratings,
'failures': failures,
}
outputs = get_model_counts()
def map_fn(k, v):
if k in ["total_models", 'failures']:
return v
elif k in ["creator", "allow_commercial_use"]:
return Counter(v)
elif k in ["ratings"]:
return sum(v) / len(v)
else:
return {
"sum": sum(v),
"avg": round(sum(v) / len(v), 3)
}
stats = {k: map_fn(k, v) for k,v in outputs.items()}
stats["num_creators"] = len(stats.pop("creator"))
for k in ["total_models", "download_count", "comment_count", "rating_count", "tipped_amount_count", "favorite_count"]:
if isinstance(stats[k], dict):
stats[f"{k}_per_creator"] = round(stats[k]["sum"] / stats["num_creators"], 2)
else:
stats[f"{k}_per_creator"] = round(stats[k] / stats["num_creators"], 2)
filename = "civitai_stats.json"
with open(filename, 'w') as file:
json.dump(stats, file, indent=4)
|