
    Fd	                    V    d Z ddlmZ ddlmZ ddlZddlmZ ddlm	Z	 	 dddZ
ddZdS )zCFunctions for counting the number of tokens in a message or string.    )annotations)ListN)Message)loggergpt-3.5-turbo-0301messagesList[Message]modelstrreturnintc                   |                     d          rd}d}d}n/|                     d          rd}d}d}nt          d| d          	 t          j        |          }n8# t          $ r+ t          j        d	           t          j        d
          }Y nw xY wd}| D ]c}||z  }|                                	                                D ]5\  }}	|t          |                    |	                    z  }|dk    r||z  }6d|dz  }|S )a  
    Returns the number of tokens used by a list of messages.

    Args:
        messages (list): A list of messages, each of which is a dictionary
            containing the role and content of the message.
        model (str): The name of the model to use for tokenization.
            Defaults to "gpt-3.5-turbo-0301".

    Returns:
        int: The number of tokens used by the list of messages.
    zgpt-3.5-turbo   zgpt-4      z4count_message_tokens() is not implemented for model z{.
 See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.z5Warning: model not found. Using cl100k_base encoding.cl100k_baser   name)
startswithNotImplementedErrortiktokenencoding_for_modelKeyErrorr   warnget_encodingrawitemslenencode)
r   r
   tokens_per_messagetokens_per_nameencoding_modelencoding
num_tokensmessagekeyvalues
             SC:\Users\Administrator\Downloads\Auto-GPT-master\autogpt\llm\utils\token_counter.pycount_message_tokensr)      sk    (( 
 	 (			'	"	" 	
 !D5 D D D
 
 	

8.~>> 8 8 8KLLL(778 J . .((
!++----// 	. 	.JC#hooe44555Jf}}o-
	. !OJs   A" "2BBstring
model_namec                n    t          j        |          }t          |                    |                     S )a  
    Returns the number of tokens in a text string.

    Args:
        string (str): The text string.
        model_name (str): The name of the encoding to use. (e.g., "gpt-3.5-turbo")

    Returns:
        int: The number of tokens in the text string.
    )r   r   r   r   )r*   r+   r#   s      r(   count_string_tokensr-   <   s.     *:66Hxv&&'''    )r   )r   r	   r
   r   r   r   )r*   r   r+   r   r   r   )__doc__
__future__r   typingr   r   autogpt.llm.baser   autogpt.logsr   r)   r-    r.   r(   <module>r5      s    I I " " " " " "        $ $ $ $ $ $       +?- - - - -`( ( ( ( ( (r.   