% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/get_tokens.R
\name{get_tokens}
\alias{get_tokens}
\title{Converts text to tokens}
\usage{
get_tokens(text, model)
}
\arguments{
\item{text}{a character string to encode to tokens, can be a vector}

\item{model}{a model to use for tokenization, either a model name, e.g., \verb{gpt-4o}
or a tokenizer, e.g., \code{o200k_base}.
See also \href{https://github.com/zurawiki/tiktoken-rs/blob/main/tiktoken-rs/src/tokenizer.rs}{available tokenizers}.}
}
\value{
a vector of tokens for the given text as integer
}
\description{
Converts text to tokens
}
\examples{
get_tokens("Hello World", "gpt-4o")
get_tokens("Hello World", "o200k_base")
get_tokens("Hello World", "gpt-5.")
get_tokens("Hello World", "text-embedding-3-small")
}
\seealso{
\code{\link[=model_to_tokenizer]{model_to_tokenizer()}}, \code{\link[=decode_tokens]{decode_tokens()}}
}
