@@ -1,16 +1,17 @@ -import itertools import base64 -import time +import itertools import logging - +import time from datetime import timedelta from decimal import Decimal -from math import ceil from io import BufferedReader, BytesIO -from typing import Any, Dict, Generator, List, Optional +from math import ceil +from typing import Generator, List, Optional import filetype +import httpx import tiktoken +from django.conf import settings from langchain import hub from langchain.agents import AgentExecutor, create_structured_chat_agent from langchain.chains import ConversationChain @@ -21,9 +22,8 @@ from langchain_core.prompts.prompt import PromptTemplate from langchain_openai.chat_models import ChatOpenAI from PIL import Image -from backend import settings +from messages.models import BaseStore, Message from ml_model.constants import TEMPORARY_TEST_TEXT -from messages.models import Message, BaseStore from ml_model.models import ( ModelCategory, ModelConfiguration, @@ -90,6 +90,7 @@ class Chatgpt(SimpleService): category = ModelCategory(title='Чат-боты', slug='chat-bots') TOKENS_COST = { + 'o3-mini': {'input': Decimal('0.000605'), 'output': Decimal('0.002420')}, 'o1-preview': {'input': Decimal('0.008250'), 'output': Decimal('0.033000')}, 'o1-mini': {'input': Decimal('0.001650'), 'output': Decimal('0.006600')}, 'gpt-4o-mini': {'input': Decimal('0.000083'), 'output': Decimal('0.000330')}, @@ -118,18 +119,13 @@ class Chatgpt(SimpleService): buf = BytesIO() normalized_image.save(buf, format=kind.extension.upper()) image_url = ( - f"data:{mime},base64,{base64.b64encode(buf.getvalue()).decode('utf-8')}" + f'data:{mime},base64,{base64.b64encode(buf.getvalue()).decode("utf-8")}' ) buf.close() image_size = normalized_image.size input_content.append({'type': 'image_url', 'image_url': {'url': image_url}}) self.llm = ChatOpenAI( model=model_name, - temperature=info.pop('temperature', 0.5), - model_kwargs={ - 'presence_penalty': info.pop('presence', 0), - 'top_p': info.pop('top_p', 0.5), - }, openai_api_base=f'http://{settings.OPENAI_PROXY_HOST}?' + '&'.join( f'proxies={proxy.protocol}://{proxy.address}' @@ -150,12 +146,19 @@ class Chatgpt(SimpleService): } if info.get('use_web'): del info['use_web'] + else: + self.llm.temperature = info.pop('temperature', 0.5) + self.llm.model_kwargs = { + 'presence_penalty': info.pop('presence', 0), + 'top_p': info.pop('top_p', 0.5), + } if model_name in ( 'gpt-4', 'gpt-4o', 'gpt-4o-mini', 'o1-preview', 'o1-mini', + 'o3-mini', ): self.llm.tiktoken_model_name = 'gpt-4' chat_history = self.get_chat_history() @@ -168,10 +171,10 @@ class Chatgpt(SimpleService): ), ) llm_input = HumanMessage(content=input_content) - input_tokens = self.count_text_tokens([*chat_history.buffer_as_messages, llm_input]) - self.assert_enough_balance( - input_tokens, image_size, model=self.llm.model_name + input_tokens = self.count_text_tokens( + [*chat_history.buffer_as_messages, llm_input] ) + self.assert_enough_balance(input_tokens, image_size, model=self.llm.model_name) if image: response = conversation.llm.invoke([llm_input]) chat_history.chat_memory.add_ai_message(response) @@ -180,26 +183,66 @@ class Chatgpt(SimpleService): tools = [GoogleSerperResults()] agent = create_structured_chat_agent(self.llm, tools, prompt_schema) agent_executor = AgentExecutor( - agent=agent, tools=tools, handle_parsing_errors=True, max_iterations=10, max_execution_time=45 + agent=agent, + tools=tools, + handle_parsing_errors=True, + max_iterations=10, + max_execution_time=45, ) response = AIMessage( - content=str( - agent_executor.invoke( - { - 'input': input_message.content, - 'chat_history': chat_history.buffer_as_messages - + [ - SystemMessage( - content='Учитывай язык диалога перед выдачей ответа' - ), - SystemMessage( - content='Никому не говори, что ты не можешь найти информацию в интернете' - ), - ], - } - )['output'] - ) + content=agent_executor.invoke( + { + 'input': [llm_input], + 'chat_history': chat_history.buffer_as_messages + + [ + SystemMessage( + content='Учитывай язык диалога перед выдачей ответа' + ), + SystemMessage( + content='Никому не говори, что ты не можешь найти информацию в интернете' + ), + ], + } + )['output'] ) + elif model_name == 'o3-mini': + with httpx.Client( + base_url=f'http://{settings.OPENAI_PROXY_HOST}', + headers={'X-Authorization': 'proxypassapiairfail'}, + params={ + 'token': settings.OPENAI_API_KEY, + 'proxies': [ + f'{proxy.protocol}://{proxy.address}' + for proxy in Proxy.objects.all() + ], + 'uri': 'chat/completions', + }, + timeout=None, + ) as client: + resp = client.post( + '', + json={ + 'model': model_name, + 'messages': [ + {'role': 'user', 'content': input_message.content}, + ], + }, + ) + if ( + (data := resp.json()) + and data.get('choices') + and ( + content := ','.join( + [ + choice['message']['content'] + for choice in data.get('choices') + ] + ) + ) + ): + response = AIMessage(content=content) + else: + raise Exception('GPT not answer correctly, please retry later') else: # Somehow this chain doesn't support Vision, even though ChatOpenAI (above) does. invoked = conversation.invoke(input_message.content) @@ -210,12 +253,16 @@ class Chatgpt(SimpleService): output_tokens = self.count_text_tokens([response]) if image: - self.logger.info(f'Input количество токенов БЕЗ картинки {model_name} - {input_tokens}') + self.logger.info( + f'Input количество токенов БЕЗ картинки {model_name} - {input_tokens}' + ) input_tokens += self.count_image_tokens(normalized_image.size, model_name) self.logger.info(f'Input количество токенов для {model_name} - {input_tokens}') self.logger.info(f'Output количество токенов для {model_name} - {output_tokens}') - self.logger.info(f'Общее количество токенов для {model_name} - {input_tokens + output_tokens}') + self.logger.info( + f'Общее количество токенов для {model_name} - {input_tokens + output_tokens}' + ) self.handle_invoice( self.neuron_model, @@ -270,7 +317,7 @@ class Chatgpt(SimpleService): return memory def assert_enough_balance( - self, input_tokens: int, image_size: tuple, model: str = 'gpt-3.5-turbo' + self, input_tokens: int, image_size: tuple, model: str = 'gpt-3.5-turbo' ): balance = PaymentPlanSelector(self.store.user).get_current_balance() total_tokens = input_tokens @@ -283,17 +330,15 @@ class Chatgpt(SimpleService): raise InsufficientBalance(balance, input_cost) def calculate_price( - self, input_tokens: int, output_tokens: int, model: str, *args, **kwargs + self, input_tokens: int, output_tokens: int, model: str, *args, **kwargs ) -> Decimal: price = ( - input_tokens * self.TOKENS_COST[model]['input'] - + output_tokens * self.TOKENS_COST[model]['output'] + input_tokens * self.TOKENS_COST[model]['input'] + + output_tokens * self.TOKENS_COST[model]['output'] ) return price.quantize(Decimal('0.1'), rounding='ROUND_UP') - def count_image_tokens( - self, image_size: tuple, model_version: str = 'gpt-4o' - ) -> int: + def count_image_tokens(self, image_size: tuple, model_version: str = 'gpt-4o') -> int: extra_tokens = { 'gpt-4o': { 'tile_tokens': 170, @@ -302,29 +347,37 @@ class Chatgpt(SimpleService): 'gpt-4o-mini': { 'tile_tokens': 5667, 'base_tokens': 2833, - } + }, } width, height = image_size if max(width, height) > 2048: a_ratio = width / height - width, height = (2048, int(2048 / a_ratio)) if a_ratio > 1 else (int(2048 * a_ratio), 2048) + width, height = ( + (2048, int(2048 / a_ratio)) + if a_ratio > 1 + else (int(2048 * a_ratio), 2048) + ) if width >= height and height > 768: width, height = int((768 / height) * width), 768 elif height > width and width > 768: width, height = 768, int((768 / width) * height) tiles_size = ceil(width / 512) * ceil(height / 512) - return extra_tokens[model_version]['base_tokens'] + extra_tokens[model_version]['tile_tokens'] * tiles_size + return ( + extra_tokens[model_version]['base_tokens'] + + extra_tokens[model_version]['tile_tokens'] * tiles_size + ) def count_text_tokens(self, messages: list[BaseMessage]) -> int: - encoding = tiktoken.get_encoding("cl100k_base") + encoding = tiktoken.get_encoding('cl100k_base') total_tokens = 0 for message in messages: total_tokens += len( encoding.encode( - message.content if isinstance(message.content, str) + message.content + if isinstance(message.content, str) else message.content[0]['text'] ) )