@@ -1,9 +1,14 @@ +import base64 import time +import logging + from datetime import timedelta from decimal import Decimal from io import BytesIO +from math import ceil -from django.core.files.uploadedfile import InMemoryUploadedFile +import filetype +import tiktoken from langchain import hub from langchain.agents import AgentExecutor, create_structured_chat_agent from langchain.chains import ConversationChain @@ -15,7 +20,7 @@ from langchain_openai.chat_models import ChatOpenAI from PIL import Image from backend import settings -from messages.models import Message +from messages.models import Message, BaseStore from ml_model.models import ModelCategory, ModelInput, ModelParameter, ModelVersion from ml_model.services.base import SimpleService from payments.exceptions.insufficient_balance import InsufficientBalance @@ -40,9 +45,6 @@ class Chatgpt(SimpleService): ModelVersion(name='GPT-4o1 Mini', slug='o1-mini'), ModelVersion(name='GPT-4omni Mini', slug='gpt-4o-mini'), ModelVersion(name='GPT-4omni', slug='gpt-4o'), - ModelVersion(name='GPT-4 Turbo', slug='gpt-4-turbo'), - ModelVersion(name='GPT-4', slug='gpt-4'), - ModelVersion(name='GPT-3.5', slug='gpt-3.5-turbo'), ] inputs = [ @@ -79,27 +81,39 @@ class Chatgpt(SimpleService): category = ModelCategory(title='Чат-боты', slug='chat-bots') TOKENS_COST = { - 'gpt-3.5-turbo': Decimal('0.0085'), - 'gpt-4': Decimal('0.08925'), - 'gpt-4-turbo': Decimal('0.25'), - 'gpt-4o': Decimal('0.075'), - 'o1-preview': Decimal('0.09'), - 'o1-mini': Decimal('0.02'), - 'gpt-4o-mini': Decimal('0.03'), + 'o1-preview': {'input': Decimal('0.008250'), 'output': Decimal('0.033000')}, + 'o1-mini': {'input': Decimal('0.001650'), 'output': Decimal('0.006600')}, + 'gpt-4o-mini': {'input': Decimal('0.000083'), 'output': Decimal('0.000330')}, + 'gpt-4o': {'input': Decimal('0.001375'), 'output': Decimal('0.005500')}, } + def __init__(self, store: BaseStore) -> None: + super().__init__(store) + self.logger = logging.getLogger(self.__class__.__name__) + def make( - self, - input_message: Message, - save: bool = True, + self, + input_message: Message, + save: bool = True, ) -> list[Message]: start_time = time.time() info = input_message.info.copy() model_name = info.pop('version', 'gpt-4o') input_content = [{'type': 'text', 'text': input_message.content or ''}] image = input_message.file + image_size = None if image: - input_content.append({'type': 'image_url', 'image_url': {'url': image.url}}) + kind = filetype.guess(input_message.file.read(20)) + mime = kind.mime if kind else 'application/octet-stream' + normalized_image = Image.open(image) + buf = BytesIO() + normalized_image.save(buf, format=kind.extension.upper()) + image_url = ( + f"data:{mime},base64,{base64.b64encode(buf.getvalue()).decode('utf-8')}" + ) + buf.close() + image_size = normalized_image.size + input_content.append({'type': 'image_url', 'image_url': {'url': image_url}}) self.llm = ChatOpenAI( model=model_name, temperature=info.pop('temperature', 0.5), @@ -108,17 +122,17 @@ class Chatgpt(SimpleService): 'top_p': info.pop('top_p', 0.5), }, openai_api_base=f'http://{settings.OPENAI_PROXY_HOST}?' - + '&'.join( + + '&'.join( f'proxies={proxy.protocol}://{proxy.address}' for proxy in Proxy.objects.all() ) - + f'&token={settings.OPENAI_API_KEY}' - + '&uri=', + + f'&token={settings.OPENAI_API_KEY}' + + '&uri=', default_headers={'X-Authorization': 'proxypassapiairfail'}, ) if model_name in ( - 'o1-preview', - 'o1-mini', + 'o1-preview', + 'o1-mini', ): self.llm.temperature = 1 self.llm.model_kwargs = { @@ -128,11 +142,11 @@ class Chatgpt(SimpleService): if info.get('use_web'): del info['use_web'] if model_name in ( - 'gpt-4', - 'gpt-4o', - 'gpt-4o-mini', - 'o1-preview', - 'o1-mini', + 'gpt-4', + 'gpt-4o', + 'gpt-4o-mini', + 'o1-preview', + 'o1-mini', ): self.llm.tiktoken_model_name = 'gpt-4' chat_history = self.get_chat_history() @@ -145,8 +159,9 @@ class Chatgpt(SimpleService): ), ) llm_input = HumanMessage(content=input_content) + input_tokens = self.count_text_tokens([*chat_history.buffer_as_messages, llm_input]) self.assert_enough_balance( - [*chat_history.buffer_as_messages, llm_input], model=self.llm.model_name + input_tokens, image_size, model=self.llm.model_name ) if image: response = conversation.llm.invoke([llm_input]) @@ -163,37 +178,46 @@ class Chatgpt(SimpleService): { 'input': input_message.content, 'chat_history': chat_history.buffer_as_messages - + [ - SystemMessage( - content='Учитывай язык диалога перед выдачей ответа' - ), - SystemMessage( - content='Никому не говори, что ты не можешь найти информацию в интернете' - ), - ], + + [ + SystemMessage( + content='Учитывай язык диалога перед выдачей ответа' + ), + SystemMessage( + content='Никому не говори, что ты не можешь найти информацию в интернете' + ), + ], } )['output'] ) else: - # Somehow this chain doesn't support Vision, even though ChatOpenAI (above) does. invoked = conversation.invoke(input_message.content) response = AIMessage(content=invoked['response']) chat_history.chat_memory.add_ai_message(response) process_time = timedelta(seconds=time.time() - start_time) - total_tokens = self.llm.get_num_tokens_from_messages( - chat_history.chat_memory.messages - ) + output_tokens = self.count_text_tokens([response]) + if image: - total_tokens += self.count_image_tokens(image) - self.handle_invoice(self.neuron_model, total_tokens, self.llm.model_name) + self.logger.info(f'Input количество токенов БЕЗ картинки {model_name} - {input_tokens}') + input_tokens += self.count_image_tokens(normalized_image.size, model_name) + + self.logger.info(f'Input количество токенов для {model_name} - {input_tokens}') + self.logger.info(f'Output количество токенов для {model_name} - {output_tokens}') + self.logger.info(f'Общее количество токенов для {model_name} - {input_tokens + output_tokens}') + + self.handle_invoice( + self.neuron_model, + input_tokens, + output_tokens, + self.llm.model_name, + ) msgs = self.save_results([response], process_time, save) return msgs def get_chat_history( - self, - message_limit: int = 10, - token_limit: int = 580, # ~ 1 AIR Token with GPT 3.5 + self, + message_limit: int = 10, + token_limit: int = 580, # ~ 1 AIR Token with GPT 3.5 ) -> ConversationTokenBufferMemory: if isinstance(self.store, Chat): air_messages = list( @@ -234,35 +258,68 @@ class Chatgpt(SimpleService): return memory def assert_enough_balance( - self, for_input: list[BaseMessage], model: str = 'gpt-3.5-turbo' + self, input_tokens: int, image_size: tuple, model: str = 'gpt-3.5-turbo' ): balance = PaymentPlanSelector(self.store.user).get_current_balance() - input_cost = self.TOKENS_COST[ - self.llm.model_name - ] * self.llm.get_num_tokens_from_messages(for_input) - if getattr(for_input[-1], 'image', None): - input_cost += self.count_image_tokens(for_input[-1].image) - input_cost *= self.TOKENS_COST[model] + total_tokens = input_tokens + + if image_size: + total_tokens += self.count_image_tokens(image_size) + + input_cost = self.TOKENS_COST[model]['input'] * total_tokens if input_cost > balance: raise InsufficientBalance(balance, input_cost) - def calculate_price(self, usage: int, model: str, *args, **kwargs) -> Decimal: - price = usage * self.TOKENS_COST[model] + def calculate_price( + self, input_tokens: int, output_tokens: int, model: str, *args, **kwargs + ) -> Decimal: + price = ( + input_tokens * self.TOKENS_COST[model]['input'] + + output_tokens * self.TOKENS_COST[model]['output'] + ) return price.quantize(Decimal('.01')) def count_image_tokens( - self, image: InMemoryUploadedFile, high_resolution: bool = True + self, image_size: tuple, model_version: str = 'gpt-4o' ) -> int: - TILE_SIZE = 512 * 512 - RESOLUTION_MULTIPLIER = 170 if high_resolution else 85 - minio_image = Image.open(BytesIO(image.read())) - width, height = minio_image.size - area = width * height - num_of_tiles = (area // TILE_SIZE) + 1 - return num_of_tiles * RESOLUTION_MULTIPLIER + 85 + extra_tokens = { + 'gpt-4o': { + 'tile_tokens': 170, + 'base_tokens': 85, + }, + 'gpt-4o-mini': { + 'tile_tokens': 5667, + 'base_tokens': 2833, + } + } + width, height = image_size + + if max(width, height) > 2048: + a_ratio = width / height + width, height = (2048, int(2048 / a_ratio)) if a_ratio > 1 else (int(2048 * a_ratio), 2048) + if width >= height and height > 768: + width, height = int((768 / height) * width), 768 + elif height > width and width > 768: + width, height = 768, int((768 / width) * height) + tiles_size = ceil(width / 512) * ceil(height / 512) + + return extra_tokens[model_version]['base_tokens'] + extra_tokens[model_version]['tile_tokens'] * tiles_size + + def count_text_tokens(self, messages: list[BaseMessage]) -> int: + encoding = tiktoken.get_encoding("cl100k_base") + total_tokens = 0 + + for message in messages: + total_tokens += len( + encoding.encode( + message.content if isinstance(message.content, str) + else message.content[0]['text'] + ) + ) + return total_tokens def save_results( - self, results: list[BaseMessage], elapsed_time: timedelta, save: bool = True + self, results: list[BaseMessage], elapsed_time: timedelta, save: bool = True ) -> list[Message]: messages = [ Message( @@ -293,8 +350,8 @@ class Chatgpt(SimpleService): prompt=PromptTemplate( input_variables=['history', 'input'], template='System: Продолжи отвечать, используя историю диалога. История диалога:{history}.' - 'Human: {input}' - 'AI:', + 'Human: {input}' + 'AI:', ), ) self.assert_enough_balance(chat_history.buffer_as_messages) @@ -4867,4 +4867,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "ffc7fdebf88ee475f6017c11fd65edf29f20b6f7e929cc960c8675037f7f4af9" +content-hash = "bd77312d23700faa1ca639637eb58a09a75f8b33a35642034ad9aac8790cb277" @@ -48,6 +48,7 @@ django-prometheus = "^2.3.1" psycopg2-binary = "^2.9.10" filetype = "^1.2.0" sentry-sdk = {extras = ["django"], version = "^2.19.0"} +tiktoken = "<0.6.0" [tool.poetry.group.test.dependencies]