DuyTa's picture
Upload folder using huggingface_hub
bc20498 verified
raw
history blame
410 Bytes
/**
* Tokenize subcontent.
*
* @param {Array<Event>} events
* List of events.
* @returns {boolean}
* Whether subtokens were found.
*/
export function subtokenize(
events: Array<import('micromark-util-types').Event>
): boolean
export type Chunk = import('micromark-util-types').Chunk
export type Event = import('micromark-util-types').Event
export type Token = import('micromark-util-types').Token