@@ -16,6 +16,7 @@ from deepl.translator import TextResult from requests import Response from backend import settings +from ml_model.utils import count_openrouter_tokens from poller.models import Proxy logger = logging.getLogger(__name__) @@ -143,7 +144,20 @@ def openrouter_run(version: str, messages: list, callback_data: dict, model_name answer = f'**Рассуждение:**\n\n{reasoning}\n\n**Основная мысль:**\n\n{content}' elif content: answer = content - return re.sub(r'\\+["n*]', '', answer), data['usage']['prompt_tokens'], data['usage']['completion_tokens'] + if data.get('choices')[0].get('error') is not None: + error = json.loads( + data['choices'][0]['error']['metadata'].replace("'", '"') + ).get('raw', {}).get('type') + if error in ('overloaded_error',): + input_tokens, output_tokens = count_openrouter_tokens(model_name, messages, content + reasoning) + else: + input_tokens = data['usage']['prompt_tokens'] + output_tokens = data['usage']['completion_tokens'] + return ( + re.sub(r'\\+["n*]', '', answer), + input_tokens, + output_tokens + ) logger.error(f'Error occured via model {model_name}. Data: {resp.content}') raise Exception(f'No answer from {model_name}, please retry later') @@ -1,5 +1,7 @@ +import tiktoken + from random import randint -from typing import Literal +from typing import Literal, List, Dict, Any, Tuple from authentication.models import CustomUserModel from authentication.selectors.account_status_selector import ( @@ -12,12 +14,12 @@ from authentication.selectors.business_account_selector import ( def random_with_N_digits(n): range_start = 10 ** (n - 1) - range_end = (10**n) - 1 + range_end = (10 ** n) - 1 return randint(range_start, range_end) def check_account_type( - user: CustomUserModel, + user: CustomUserModel, ) -> Literal['business_host'] | Literal['business_account'] | Literal['regular'] | Literal['business_admin']: status = AccountStatusSelector(user) if status.is_business_host(): @@ -30,3 +32,26 @@ def check_account_type( return 'business_admin' return 'business_account' + + +def count_openrouter_tokens(model_name: str, messages: List[Dict[str, Any]], output: str) -> Tuple[int, int]: + """A function for count tokens for OpenRouter Neuron Models""" + encodings = { + 'Qwen': 'cl100k_base', + 'Deepseek': 'cl100k_base', + 'Claude': 'r50k_base', + 'Perplexity': 'cl100k_base', + 'Mistral': 'r50k_base', + 'LLaMA': 'cl100k_base', + 'Grok': 'r50k_base', + 'Gemini': 'cl100k_base', + } + encoding = tiktoken.get_encoding(encodings[model_name]) + input_tokens = 100 if model_name == 'LLaMA' else 0 + for message in messages: + if isinstance(message['content'], list): + input_tokens += len(encoding.encode(message['content'][0]['text'])) + else: + input_tokens += len(encoding.encode(message['content'])) + output_tokens = len(encoding.encode(output)) + return (input_tokens, output_tokens)