aboutsummaryrefslogtreecommitdiff
path: root/tests/unit/test_token_counter.py
diff options
context:
space:
mode:
Diffstat (limited to 'tests/unit/test_token_counter.py')
-rw-r--r--tests/unit/test_token_counter.py55
1 files changed, 0 insertions, 55 deletions
diff --git a/tests/unit/test_token_counter.py b/tests/unit/test_token_counter.py
deleted file mode 100644
index feed18b13..000000000
--- a/tests/unit/test_token_counter.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import pytest
-
-from autogpt.llm.base import Message
-from autogpt.llm.utils import count_message_tokens, count_string_tokens
-
-
-def test_count_message_tokens():
- messages = [
- Message("user", "Hello"),
- Message("assistant", "Hi there!"),
- ]
- assert count_message_tokens(messages) == 17
-
-
-def test_count_message_tokens_empty_input():
- """Empty input should return 3 tokens"""
- assert count_message_tokens([]) == 3
-
-
-def test_count_message_tokens_invalid_model():
- """Invalid model should raise a NotImplementedError"""
- messages = [
- Message("user", "Hello"),
- Message("assistant", "Hi there!"),
- ]
- with pytest.raises(NotImplementedError):
- count_message_tokens(messages, model="invalid_model")
-
-
-def test_count_message_tokens_gpt_4():
- messages = [
- Message("user", "Hello"),
- Message("assistant", "Hi there!"),
- ]
- assert count_message_tokens(messages, model="gpt-4-0314") == 15
-
-
-def test_count_string_tokens():
- """Test that the string tokens are counted correctly."""
-
- string = "Hello, world!"
- assert count_string_tokens(string, model_name="gpt-3.5-turbo-0301") == 4
-
-
-def test_count_string_tokens_empty_input():
- """Test that the string tokens are counted correctly."""
-
- assert count_string_tokens("", model_name="gpt-3.5-turbo-0301") == 0
-
-
-def test_count_string_tokens_gpt_4():
- """Test that the string tokens are counted correctly."""
-
- string = "Hello, world!"
- assert count_string_tokens(string, model_name="gpt-4-0314") == 4