File size: 6,352 Bytes
a28bbcf |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 |
import orjson
import sqlite3
import requests
import time
import math
import logging
CACHE_EXPIRE_TIME = 60 * 60 * 12
logger = logging.getLogger('EmojiStore')
class EmojiStore:
def __init__(self, db, **kwargs):
self.db: sqlite3.Connection = db
self.db.row_factory = sqlite3.Row
cur = self.db.cursor()
cur.execute('CREATE TABLE IF NOT EXISTS emoji_cache(host TEXT, data TEXT, last_updated INTEGER)')
cur.close()
self.emoji_cache = {}
if kwargs.get('session'):
self.session = kwargs['session']
else:
self.session = requests.Session()
self.session.headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36'
def _generate_emoji_url(self, host, emoji: dict):
if emoji.get('url'):
return emoji['url']
else:
# v>=13?
return f'https://{host}/emoji/{emoji["name"]}.webp'
def _fetch_nodeinfo(self, host):
r = self.session.get(f'https://{host}/.well-known/nodeinfo')
if r.status_code != 200:
logger.getChild('fetch_nodeinfo').error(f'Failed to fetch nodeinfo for {host} (well-known/nodeinfo)')
raise Exception(f'Failed to fetch nodeinfo for {host}')
res = orjson.loads(r.content)
if res.get('links'):
for link in res['links']:
if link['rel'].endswith('nodeinfo.diaspora.software/ns/schema/2.0'):
r2 = self.session.get(link['href'])
if r2.status_code != 200:
logger.getChild('fetch_nodeinfo').error(f'Failed to fetch nodeinfo for {host} (nodeinfo)')
raise Exception(f'Failed to fetch nodeinfo for {host}')
return orjson.loads(r2.content)
logger.getChild('fetch_nodeinfo').error(f'Failed to fetch nodeinfo for {host}')
raise Exception(f'Failed to fetch nodeinfo for {host}')
def _fetch_emoji_data(self, host):
logger.getChild('fetch_emoji_data').info(f'Fetching emoji data for {host}')
try:
ni = self._fetch_nodeinfo(host)
r = self.session.post(f'https://{host}/api/meta', headers={'Content-Type': 'application/json'}, data=b'{}')
if r.status_code != 200 and r.status_code != 404:
logger.getChild('fetch_emoji_data').error(f'Failed to fetch emoji data for {host} (api/meta)')
raise Exception(f'Failed to fetch emoji data for {host}')
if r.status_code != 404:
meta = orjson.loads(r.content)
v = meta['version'].split('.')
# Misskey v13以降は別エンドポイントに問い合わせ
if ni['software']['name'] == 'misskey' and int(v[0]) >= 13:
r2 = self.session.post(f'https://{host}/api/emojis', headers={'Content-Type': 'application/json'}, data=b'{}')
if r2.status_code != 200:
logger.getChild('fetch_emoji_data').error(f'Failed to fetch emoji data for {host} (Misskey v13)')
raise Exception(f'Failed to fetch emoji data for {host} (Misskey v13)')
return orjson.loads(r2.content)['emojis']
else:
return meta['emojis']
else:
# Mastodon/Pleroma?
r3 = self.session.get(f'https://{host}/api/v1/custom_emojis')
if r3.status_code != 200:
logger.getChild('fetch_emoji_data').error(f'Failed to fetch emoji data for {host} (mastodon, pleroma)')
raise Exception(f'Failed to fetch emoji data for {host} (mastodon, pleroma)')
res = orjson.loads(r3.content)
# Misskey形式に変換
return [{'name': x['shortcode'], 'url': x['static_url'], 'aliases': [''], 'category': ''} for x in res]
except:
return []
def _download(self, host):
emoji_data = self._fetch_emoji_data(host)
cur = self.db.cursor()
cur.execute('REPLACE INTO emoji_cache(host, data, last_updated) VALUES (?, ?, ?)', (host, orjson.dumps(emoji_data), math.floor(time.time())))
self.db.commit()
self.emoji_cache[host] = emoji_data
def _load(self, host) -> list:
emojis = []
if host in self.emoji_cache.keys():
emojis = self.emoji_cache[host]
else:
cur = self.db.cursor()
cur.execute('SELECT * FROM emoji_cache WHERE host = ?', (host,))
row = cur.fetchone()
if row is None:
logger.getChild('load').error(f'emoji data not found. fetching')
self._download(host)
return self._load(host)
else:
expire = CACHE_EXPIRE_TIME
# 前回取得失敗してる？
if row['data'] == '[]':
expire = 60 * 5
if math.floor(time.time()) - row['last_updated'] > expire:
logger.getChild('load').error(f'emoji cache expired. refreshing')
self._download(host)
return self._load(host)
self.emoji_cache[host] = orjson.loads(row['data'])
emojis = self.emoji_cache[host]
return emojis
# ----------------------
def refresh(self, host):
self._download(host)
def find_by_keyword(self, host, k) -> list:
emojis = self._load(host)
res = []
for emoji in emojis:
if k in emoji['name'].lower():
res.append({'name': emoji['name'], 'url': self._generate_emoji_url(host, emoji)})
return res
def find_by_alias(self, host, t) -> list:
emojis = self._load(host)
res = []
for emoji in emojis:
if t in emoji['aliases']:
res.append({'name': emoji['name'], 'url': self._generate_emoji_url(host, emoji)})
return res
def get(self, host, name):
emojis = self._load(host)
for emoji in emojis:
if emoji['name'] == name:
return {'name': emoji['name'], 'url': self._generate_emoji_url(host, emoji)}
return None |