Skip to content

amazon_bedrock_tokenizer

AmazonBedrockTokenizer

Bases: BaseTokenizer

Source code in griptape/tokenizers/amazon_bedrock_tokenizer.py 
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
@define()
class AmazonBedrockTokenizer(BaseTokenizer):
    MODEL_PREFIXES_TO_MAX_INPUT_TOKENS = {
        "anthropic.claude-3": 200000,
        "anthropic.claude-v2:1": 200000,
        "anthropic.claude": 100000,
        "cohere.command-r": 128000,
        "cohere.embed": 512,
        "cohere.command": 4000,
        "cohere": 1024,
        "ai21": 8192,
        "meta.llama3-8b-instruct": 8000,
        "meta.llama3-70b-instruct": 8000,
        "meta.llama3-2-1b-instruct": 131000,
        "meta.llama3-2-3b-instruct": 131000,
        "meta.llama3": 128000,
        "mistral.large-2407": 128000,
        "mistral.mistral": 32000,
        "mistral.mixtral": 32000,
        "amazon.nova-micro-v1": 128000,
        "amazon.nova": 300000,
        "amazon.titan-embed-image": 128000,
        "amazon.titan-embed-text": 8000,
        "amazon.titan-text-express-v1": 8000,
        "amazon.titan-text-lite-v1": 4000,
        "amazon.titan-text-premier-v1": 32000,
    }
    MODEL_PREFIXES_TO_MAX_OUTPUT_TOKENS = {
        "anthropic.claude-3-7": 8192,
        "anthropic.claude-3-5": 8192,
        "anthropic.claude": 4096,
        "cohere": 4096,
        "ai21.j2": 8191,
        "meta": 2048,
        "amazon.titan-text-lite": 4096,
        "amazon.titan-text-express": 8192,
        "amazon.titan-text-premier": 3072,
        "amazon.nova": 5000,
        "mistral.mistral": 8192,
        "mistral.mixtral": 4096,
    }

    model: str = field(kw_only=True)
    characters_per_token: int = field(default=4, kw_only=True)

    def count_tokens(self, text: str) -> int:
        return (len(text) + self.characters_per_token - 1) // self.characters_per_token

MODEL_PREFIXES_TO_MAX_INPUT_TOKENS = {'anthropic.claude-3': 200000, 'anthropic.claude-v2:1': 200000, 'anthropic.claude': 100000, 'cohere.command-r': 128000, 'cohere.embed': 512, 'cohere.command': 4000, 'cohere': 1024, 'ai21': 8192, 'meta.llama3-8b-instruct': 8000, 'meta.llama3-70b-instruct': 8000, 'meta.llama3-2-1b-instruct': 131000, 'meta.llama3-2-3b-instruct': 131000, 'meta.llama3': 128000, 'mistral.large-2407': 128000, 'mistral.mistral': 32000, 'mistral.mixtral': 32000, 'amazon.nova-micro-v1': 128000, 'amazon.nova': 300000, 'amazon.titan-embed-image': 128000, 'amazon.titan-embed-text': 8000, 'amazon.titan-text-express-v1': 8000, 'amazon.titan-text-lite-v1': 4000, 'amazon.titan-text-premier-v1': 32000} class-attribute instance-attribute

MODEL_PREFIXES_TO_MAX_OUTPUT_TOKENS = {'anthropic.claude-3-7': 8192, 'anthropic.claude-3-5': 8192, 'anthropic.claude': 4096, 'cohere': 4096, 'ai21.j2': 8191, 'meta': 2048, 'amazon.titan-text-lite': 4096, 'amazon.titan-text-express': 8192, 'amazon.titan-text-premier': 3072, 'amazon.nova': 5000, 'mistral.mistral': 8192, 'mistral.mixtral': 4096} class-attribute instance-attribute

characters_per_token = field(default=4, kw_only=True) class-attribute instance-attribute

model = field(kw_only=True) class-attribute instance-attribute

count_tokens(text)

Source code in griptape/tokenizers/amazon_bedrock_tokenizer.py 
53
54
def count_tokens(self, text: str) -> int:
    return (len(text) + self.characters_per_token - 1) // self.characters_per_token