@@ -369,6 +369,8 @@ USER_CONFIRMATION_URL = env.str('USER_CONFIRMATION_URL', default='http://localho USER_PASSWORD_RESET_URL = env.str('USER_PASSWORD_RESET_URL', default='http://localhost:3000') INVITATION_RESPONSE_URL = env.str('INVITATION_RESPONSE_URL', default='http://localhost:3000') +MAX_THREADS = env.int('MAX_THREADS', default=3) + MAIN_SITE_URL = env.str('MAIN_SITE_URL', default='http://localhost:3000') ERROR_EMAIL_RECIPIENTS = env.list( @@ -1,14 +1,16 @@ import base64 import itertools import logging +import re import subprocess import time +from concurrent.futures import ThreadPoolExecutor, as_completed + import numpy as np import openpyxl import fitz import redis -from django.db.models import QuerySet from django.utils.translation import gettext_lazy as _ from datetime import timedelta @@ -37,8 +39,6 @@ from langchain_openai.chat_models import ChatOpenAI from langchain_text_splitters import RecursiveCharacterTextSplitter from PIL import Image, UnidentifiedImageError from redis.commands.search.document import Document -from redis.commands.search.field import TextField, VectorField, TagField -from redis.commands.search.indexDefinition import IndexDefinition, IndexType from redis.commands.search.query import Query from backend import settings @@ -130,9 +130,13 @@ class Chatgpt(SimpleService): if file: file_extension = Path(file.name).suffix if file_extension == '.pdf': - chunks = self.split_text_to_chunks(self.get_pdf_data(file)) + raw_text = self.get_pdf_data(file) + text = re.sub(r'\n{2,}', '\n', raw_text) + chunks = self.split_text_to_chunks(text) elif file_extension in ('.doc', '.docx'): - chunks = self.split_text_to_chunks(self.get_word_data(file_extension, file)) + raw_text = self.get_word_data(file_extension, file) + text = re.sub(r'\n{2,}', '\n', raw_text) + chunks = self.split_text_to_chunks(text) elif file_extension == '.xlsx': chunks = self.split_text_to_chunks(self.get_xlsx_data(file)) else: @@ -256,38 +260,43 @@ class Chatgpt(SimpleService): redis_client = redis.Redis(host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=0) document_name = chunks[0].content.partition(f':{chr(10)}')[2].split(f'{chr(10)}')[0][:100] message_uid = str(self.store.messages.first().pk).replace('-', '_') - for chunk_id, chunk in enumerate(chunks): - embedding, e_total_tokens = self.get_embedding(proxy=proxy, content=chunk.content) - self.save_embeddings( - redis_client=redis_client, - message_uid=message_uid, - chunk_id=chunk_id, - text=chunk.content, - embeddings=embedding - ) + with httpx.Client( + base_url='https://api.openai.com/v1/', + proxy=f'{proxy.protocol}://{proxy.address}', + headers={'Authorization': f'Bearer {settings.OPENAI_API_KEY}'}, + timeout=600, + ) as client: + threads = [] + with ThreadPoolExecutor(max_workers=settings.MAX_THREADS) as executor: + for chunk_id, chunk in enumerate(chunks): + threads.append( + executor.submit(self.process_chunk, client, chunk, redis_client, message_uid, chunk_id) + ) + for thread in as_completed(threads): + embedding_tokens += thread.result() + query_embedding, e_total_tokens = self.get_embedding(client=client, content=input_message.content) embedding_tokens += e_total_tokens - query_embedding, e_total_tokens = self.get_embedding(proxy=proxy, content=input_message.content) - embedding_tokens += e_total_tokens - result = [ - s['section_text'] - for s in self.search_via_embeddings( - redis_client=redis_client, - message_uid=message_uid, - user_query_embeddings=query_embedding + result = [ + s['section_text'] + for s in self.search_via_embeddings( + redis_client=redis_client, + message_uid=message_uid, + user_query_embeddings=query_embedding + ) + ] + user_input = [ + SystemMessage(content=user_system_prompt), + HumanMessage(self.make_embeddings_prompt( + document_name=document_name, section_texts=result, question=input_message.content + )) + ] + input_tokens += self.count_text_tokens(user_input) + response = conversation.invoke( + {'input': user_input}, + config={'configurable': {'session_id': 'default'}}, ) - ] - user_input = [ - SystemMessage(content=user_system_prompt), - HumanMessage(self.make_embeddings_prompt( - document_name=document_name, section_texts=result, question=input_message.content - )) - ] - input_tokens += self.count_text_tokens(user_input) - response = conversation.invoke( - {'input': user_input}, - config={'configurable': {'session_id': 'default'}}, - ) - drop_redis_vectors.delay(message_uid) + drop_redis_vectors.delay(message_uid) + redis_client.close() else: input = [ SystemMessage(content=user_system_prompt), @@ -551,21 +560,44 @@ class Chatgpt(SimpleService): chunks = text_splitter.split_text(raw_text) return [HumanMessage(chunk) for chunk in chunks] - def get_embedding(self, proxy: Proxy, content: str) -> Tuple[List[float], int]: + def process_chunk( + self, client: httpx.Client, chunk: HumanMessage, redis_client: redis.Redis, message_uid:str, chunk_id: int + ) -> int: + ''' + A method for getting and saving embeddings from a single chunk + :param client: Httpx client + :param chunk: a HumanMessage object with a content as a part of a full text + :param redis_client: Redis client + :param message_uid: UID of user's message + :param chunk_id: a sequence number of a chunk + ''' + embedding, e_total_tokens = self.get_embedding(client=client, content=chunk.content) + self.save_embeddings( + redis_client=redis_client, + message_uid=message_uid, + chunk_id=chunk_id, + text=chunk.content, + embeddings=embedding + ) + return e_total_tokens + + def get_embedding(self, client: httpx.Client, content: str) -> Tuple[List[float], int]: ''' A method for converting raw text (content) into embeddings using OpenAI API request - :param proxy: Proxy settings object with protocol and address + :param client: Httpx client :param content: raw text of a chunk ''' - return self.call_openai_api( - proxy=proxy, - endpoint='embeddings', - json_data={ + response = client.post( + url="embeddings", + json={ 'model': 'text-embedding-3-large', 'input': content } ) + response.raise_for_status() + data = response.json() + return data['data'][0]['embedding'], data['usage']['total_tokens'] def save_embeddings( self, redis_client: redis.Redis, message_uid: str, chunk_id: int, text: str, embeddings: List[float] @@ -673,13 +705,6 @@ class Chatgpt(SimpleService): output_tokens = resp.json()['usage']['output_tokens'] response = AIMessage(content=content) return input_tokens, output_tokens, response - elif ( - endpoint == 'embeddings' - and (data := resp.json()) - and (embedding := data['data'][0]['embedding']) - and (tokens := data['usage']['total_tokens']) - ): - return (embedding, tokens) else: raise Exception('GPT not answer correctly, please retry later') @@ -1,6 +1,13 @@ +import re import time +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures._base import as_completed +from typing import List, Tuple + import httpx import base64 + +import redis import requests import fitz @@ -37,6 +44,8 @@ from ml_model.exceptions import GenerationException from poller.models import Proxy +from ml_model.tasks import drop_redis_vectors +from ml_model.constants import ANCHORS class Raifgpt(Chatgpt): @property @@ -56,15 +65,18 @@ class Raifgpt(Chatgpt): except Exception as exc: raise TemplateUnknownException from exc input_content = [{'type': 'text', 'text': input_message.content or ''}] + embedding_tokens = 0 file = input_message.file if file: file_extension = Path(file.name).suffix if file_extension == '.pdf': - chunks = self.split_text_to_chunks(self.get_pdf_data(file)) + raw_text = self.get_pdf_data(file) elif file_extension in ('.doc', '.docx'): - chunks = self.split_text_to_chunks(self.get_word_data(file_extension, file)) + raw_text = self.get_word_data(file_extension, file) else: raise FileExtensionNotSupported(['PDF', 'DOC', 'DOCX']) + text = re.sub(r'\n{2,}', '\n', raw_text) + chunks = self.split_text_to_chunks(text, chunk_size=1000) for proxy in Proxy.objects.all(): self.llm = ChatOpenAI( model='gpt-4o', @@ -92,17 +104,88 @@ class Raifgpt(Chatgpt): start_time = time.time() try: if file: - input = [ - SystemMessage(content=user_system_prompt), - HumanMessage(content=f'Строго используй системный промпт. Вот содержание файла по чанкам:'), - *chunks, - llm_input - ] - input_tokens = self.count_text_tokens(input) - response = conversation.invoke( - {'input': input}, - config={'configurable': {'session_id': 'default'}}, - ) + input_tokens = self.count_text_tokens([*chat_history.messages]) + if sum([len(chunk.content) for chunk in chunks]) > 40_000: + redis_client = redis.Redis(host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=0) + document_name = chunks[0].content.partition(f':{chr(10)}')[2].split(f'{chr(10)}')[0][:100] + message_uid = str(self.store.messages.first().pk).replace('-', '_') + with httpx.Client( + base_url='https://api.openai.com/v1/', + proxy=f'{proxy.protocol}://{proxy.address}', + headers={'Authorization': f'Bearer {settings.OPENAI_API_KEY}'}, + timeout=600, + ) as client: + threads = [] + with ThreadPoolExecutor(max_workers=settings.MAX_THREADS) as executor: + for chunk_id, chunk in enumerate(chunks): + threads.append( + executor.submit( + self.process_chunk, client, chunk, redis_client, message_uid, chunk_id + ) + ) + for thread in as_completed(threads): + embedding_tokens += thread.result() + if not input_message.content: + threads.clear() + anchor_embeddings = {} + with ThreadPoolExecutor(max_workers=settings.MAX_THREADS) as executor: + for identify, value in ANCHORS.items(): + threads.append(executor.submit(self.get_anchor_embedding, client, value[0], identify)) + for thread in as_completed(threads): + thread_result = thread.result() + embedding_tokens += thread_result[1] + anchor_embeddings[thread_result[2]] = thread_result[0] + threads.clear() + with ThreadPoolExecutor(max_workers=settings.MAX_THREADS) as executor: + for identify, embeddings in anchor_embeddings.items(): + threads.append( + executor.submit( + self.search_via_embeddings, + redis_client, + message_uid, + embeddings, + top_k=ANCHORS[identify][1] + ) + ) + result = [s['section_text'] for thread in as_completed(threads) for s in thread.result()] + else: + query_embedding, e_total_tokens = self.get_embedding(client=client, content=input_message.content) + embedding_tokens += e_total_tokens + result = [ + s['section_text'] + for s in self.search_via_embeddings( + redis_client=redis_client, + message_uid=message_uid, + user_query_embeddings=query_embedding, + top_k=25 + ) + ] + user_input = [ + SystemMessage(content=user_system_prompt), + HumanMessage(self.make_embeddings_prompt( + document_name=document_name, section_texts=result, question=input_message.content + )) + ] + input_tokens += self.count_text_tokens(user_input) + response = conversation.invoke( + {'input': user_input}, + config={'configurable': {'session_id': 'default'}}, + ) + drop_redis_vectors.delay(message_uid) + redis_client.close() + else: + input = [ + SystemMessage(content=user_system_prompt), + HumanMessage( + content=f'Используй системный промпт. Содержание файла: ' + f'{chunks}. Вопрос: {input_message.content}' + ) + ] + input_tokens += self.count_text_tokens(input) + response = conversation.invoke( + {'input': input}, + config={'configurable': {'session_id': 'default'}}, + ) else: response = conversation.invoke( {'input': llm_input}, @@ -116,9 +199,10 @@ class Raifgpt(Chatgpt): output_tokens = self.count_text_tokens([response]) - self.logger.info(f'Input количество токенов для gpt-4o - {input_tokens}') - self.logger.info(f'Output количество токенов для gpt-4o - {output_tokens}') - self.logger.info(f'Общее количество токенов для gpt-4o - {input_tokens + output_tokens}') + self.logger.info(f'Input количество токенов для raifgpt - {input_tokens}') + self.logger.info(f'Output количество токенов для raifgpt - {output_tokens}') + self.logger.info(f'Embedding количество токенов для raifgpt - {embedding_tokens}') + self.logger.info(f'Общее количество токенов для raifgpt - {input_tokens + output_tokens + embedding_tokens}') process_time = timedelta(seconds=time.time() - start_time) self.handle_invoice( @@ -126,7 +210,8 @@ class Raifgpt(Chatgpt): input_tokens, output_tokens, self.llm.model_name, - {} + {}, + embedding_tokens ) msgs = self.save_results([response], process_time, save) return msgs @@ -234,3 +319,47 @@ class Raifgpt(Chatgpt): return final_text.strip() or "Не удалось распознать текст" except Exception as e: return f"Не удалось обработать файл: {e}" + + def make_embeddings_prompt(self, document_name: str, section_texts: List[str], question: str) -> str: + ''' + A method for making a prompt using found embeddings + :param document_name: name of the loaded document + :param section_texts: list of sections' contents + :param question: user question + ''' + return f"""Ты — аналитик данных моей компании. + Отвечай исключительно на основе предоставленного ниже контекста. + НЕЛЬЗЯ использовать внешние знания или домыслы. + СТРОГО СЛЕДУЙ СИСТЕМНОМУ ПРОМПТУ и НЕ ВЫХОДИ за его рамки. + Нельзя отвечать "не могу помочь" — даже при нехватке данных СФОРМИРУЙ ответ на основе доступной информации. + Если данных мало — делай это явно и заполни только доступные части. + + Название файла: {document_name} + + Контекстные фрагменты: + {'\n'.join(section_texts)} + + Вопрос: + {question} + + Сформируй ПОЛНЫЙ и СТРУКТУРИРОВАННЫЙ ответ, даже если доступные данные частичные. + """ + + def get_anchor_embedding(self, client: httpx.Client, content: str, anchor: str) -> Tuple[List[float], int, str]: + ''' + A method for converting raw text (anchor content) into embeddings + using OpenAI API request + :param client: Httpx client + :param content: raw text of a chunk + :param anchor: anchor identifier + ''' + response = client.post( + url="embeddings", + json={ + 'model': 'text-embedding-3-large', + 'input': content + } + ) + response.raise_for_status() + data = response.json() + return data['data'][0]['embedding'], data['usage']['total_tokens'], anchor \ No newline at end of file @@ -57,3 +57,105 @@ Nam convallis mi a luctus laoreet. Etiam suscipit egestas posuere. Aenean eget f Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Donec rutrum viverra suscipit. Etiam nec laoreet quam. Aenean in ligula sit amet nibh consectetur porta. Cras molestie nisl a odio lacinia, ac ornare odio hendrerit. Mauris et pulvinar dolor. Morbi pretium luctus sem, ac congue eros ornare eget. Pellentesque laoreet elementum felis id mollis. Maecenas velit nulla, gravida sodales lectus in, fermentum auctor justo. Sed euismod maximus ex, vel pharetra massa ultricies vel. Curabitur interdum cursus quam ut placerat. Phasellus eleifend purus sed nisi pellentesque condimentum. Fusce finibus, felis quis mattis tempus, tortor elit tincidunt massa, non interdum tellus neque id mauris. Morbi dapibus volutpat lectus sed pellentesque. Curabitur sagittis lacinia quam eu viverra. In sit amet nunc sed urna aliquet vehicula id vel justo. Duis vel massa eleifend, tristique ante in, ullamcorper tellus. Donec egestas lacus eu libero sodales luctus. Duis id maximus arcu, non dictum felis. Cras nisl odio, viverra in arcu at, sagittis gravida augue. Aliquam quis metus vel urna finibus sodales. Praesent a porttitor magna. Integer eu est ac ligula luctus consequat. """ + +ANCHORS = { + "authors": ( + "автор|авторы|составители|подготовители|команда|коллектив|исследователь|" + "исследователи|авторский коллектив|writer|researcher|investigator|" + "contributors|исполнители|ответственные лица|authorship|авторство|" + "group|team|authorship team", + 2 + ), + "topic": ( + "тема исследования|предмет исследования|тема работы|цель исследования|" + "предмет|направление|scope|research topic|subject of study|research focus|" + "object of study|scientific problem|область исследования|problem statement", + 2 + ), + "summary": ( + "краткое содержание|основные выводы|итоги исследования|summary|conclusions|" + "executive summary|highlights|abstract|overview|synopsis|выводы|" + "резюме|summary statement", + 6 + ), + "volume": ( + "объем рынка|market size|размер рынка|объем продаж|общие показатели|" + "объем инвестиций|total volume|market volume|рыночная капитализация|" + "оборот|объем финансирования|масштаб рынка|market capacity", + 4 + ), + "forecast": ( + "прогноз|forecast|прогнозные показатели|ожидания|перспективы|outlook|" + "прогноз развития|predicted values|прогноз роста|прогноз падения|" + "future outlook|прогноз на следующий год|прогноз на 3-5 лет", + 4 + ), + "growth_drivers": ( + "драйверы роста|факторы роста|причины роста|growth drivers|" + "growth factors|catalysts|key drivers|стимулирующие факторы|" + "факторы развития|движущие силы|причины повышения|рост рынка|growth enablers", + 4 + ), + "barriers": ( + "барьеры|препятствия|ограничения|риски|сложности|ограничения рынка|" + "barriers|obstacles|challenges|risks|факторы замедления|факторы риска|" + "рисковые факторы|проблемы|тормозящие развитие|негативные факторы", + 4 + ), + "regulations": ( + "регуляторные изменения|законодательство|нормативные акты|регулирование|" + "compliance|laws|regulations|legal changes|закон|правила|постановления|" + "стандарты|регулирующие органы|политические инициативы|правовые нормы", + 4 + ), + "segmentation": ( + "сегментация|разделение рынка|сегменты|группы клиентов|customer segments|" + "market segmentation|категории|подразделения|типы клиентов|демографические" + " группы|целевые аудитории|сегментация по регионам|product segmentation", + 4 + ), + "players": ( + "игроки рынка|компании|корпорации|основные участники|конкуренты|market " + "players|key companies|competitors|поставщики|лидеры рынка|крупные компании|" + "бизнес-игроки|участники рынка|основные бренды", + 4 + ), + "quant_metrics": ( + "количественные метрики|числовые показатели|quantitative metrics|цифры|" + "data points|измерения|показатели|объемы|количество сделок|темпы роста|" + "проценты|значения|финансовые показатели|статистика", + 2 + ), + "qual_metrics": ( + "качественные метрики|качественные показатели|qualitative metrics|оценки" + "|факторы оценки|quality indicators|мнение экспертов|экспертные оценки|" + "восприятие|качественные данные|отзывы|качественный анализ", + 2 + ), + "cases": ( + "кейсы|примеры|практические примеры|case studies|examples|use cases|" + "проекты|сценарии|успешные истории|best practices|опыт применения", + 1 + ), + "charts": ( + "графики|диаграммы|charts|diagrams|visualizations|plots|иллюстрации|" + "схемы|инфографика|data visualization|charts and graphs", + 1 + ), + "tables": ( + "таблицы|data tables|таблицы данных|spreadsheets|matrices|таблицы с данными|" + "табличные данные|списки|структуры данных|табличное представление", + 1 + ), + "methodology": ( + "методология|методы исследования|approach|methodology|methods|techniques|" + "исследовательские методы|методики|способы анализа|процедура|процесс исследования", + 1 + ), + "interviews": ( + "интервью|мнения экспертов|комментарии|expert interviews|expert opinions|" + "statements|reviews|опросы|интервью с экспертами|экспертные отзывы|" + "интервьюирование|отзывы участников", + 1 + ), +} @@ -70,6 +70,13 @@ USER_PASSWORD_RESET_URL='https://app.air.fail/changePassword' INVITATION_RESPONSE_URL='https://app.air.fail/business/confirm' MAIN_SITE_URL=https://app.air.fail +# THREADS +MAX_THREADS=3 + +# REDIS +REDIS_HOST=cache-mdb +REDIS_PORT=6379 + # DEBUG USER DJANGO_SUPERUSER_EMAIL=example@root.ru DJANGO_SUPERUSER_USERNAME=root @@ -5,7 +5,7 @@ WORKDIR /code COPY pyproject.toml poetry.lock /code/ RUN --mount=type=cache,target=/root/.cache/pip pip install poetry && poetry self add poetry-plugin-export -RUN poetry export --with test --with debug --output=requirements.txt +RUN poetry export --with test --with debug --with dev --output=requirements.txt FROM python:3.12-slim @@ -2868,6 +2868,21 @@ dev = ["marshmallow[tests]", "pre-commit (>=3.5,<5.0)", "tox"] docs = ["autodocsumm (==0.2.14)", "furo (==2024.8.6)", "sphinx (==8.1.3)", "sphinx-copybutton (==0.5.2)", "sphinx-issues (==5.0.0)", "sphinxext-opengraph (==0.9.1)"] tests = ["pytest", "simplejson"] +[[package]] +name = "memory-profiler" +version = "0.61.0" +description = "A module for monitoring memory usage of a python program" +optional = false +python-versions = ">=3.5" +groups = ["dev"] +files = [ + {file = "memory_profiler-0.61.0-py3-none-any.whl", hash = "sha256:400348e61031e3942ad4d4109d18753b2fb08c2f6fb8290671c5513a34182d84"}, + {file = "memory_profiler-0.61.0.tar.gz", hash = "sha256:4e5b73d7864a1d1292fb76a03e82a3e78ef934d06828a698d9dada76da2067b0"}, +] + +[package.dependencies] +psutil = "*" + [[package]] name = "minio" version = "7.2.14" @@ -3224,6 +3239,21 @@ rsa = ["cryptography (>=3.0.0)"] signals = ["blinker (>=1.4.0)"] signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] +[[package]] +name = "objgraph" +version = "3.6.2" +description = "Draws Python object reference graphs with graphviz" +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "objgraph-3.6.2-py3-none-any.whl", hash = "sha256:8114c97712291c3ba30d882406a384d0a7651b307ea9a06e0d83836ccde85e15"}, + {file = "objgraph-3.6.2.tar.gz", hash = "sha256:00b9f2f40f7422e3c7f45a61c4dafdaf81f03ff0649d6eaec866f01030e51ad8"}, +] + +[package.extras] +ipython = ["graphviz"] + [[package]] name = "openai" version = "1.59.7" @@ -3637,6 +3667,30 @@ files = [ {file = "protobuf-4.25.5.tar.gz", hash = "sha256:7f8249476b4a9473645db7f8ab42b02fe1488cbe5fb72fddd445e0665afd8584"}, ] +[[package]] +name = "psutil" +version = "7.0.0" +description = "Cross-platform lib for process and system monitoring in Python. NOTE: the syntax of this script MUST be kept compatible with Python 2.7." +optional = false +python-versions = ">=3.6" +groups = ["dev"] +files = [ + {file = "psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25"}, + {file = "psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da"}, + {file = "psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91"}, + {file = "psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34"}, + {file = "psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993"}, + {file = "psutil-7.0.0-cp36-cp36m-win32.whl", hash = "sha256:84df4eb63e16849689f76b1ffcb36db7b8de703d1bc1fe41773db487621b6c17"}, + {file = "psutil-7.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:1e744154a6580bc968a0195fd25e80432d3afec619daf145b9e5ba16cc1d688e"}, + {file = "psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99"}, + {file = "psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553"}, + {file = "psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456"}, +] + +[package.extras] +dev = ["abi3audit", "black (==24.10.0)", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest", "pytest-cov", "pytest-xdist", "requests", "rstcheck", "ruff", "setuptools", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "vulture", "wheel"] +test = ["pytest", "pytest-xdist", "setuptools"] + [[package]] name = "psycopg2-binary" version = "2.9.10" @@ -3968,6 +4022,21 @@ dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pyte docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] +[[package]] +name = "pympler" +version = "1.1" +description = "A development tool to measure, monitor and analyze the memory behavior of Python objects." +optional = false +python-versions = ">=3.6" +groups = ["dev"] +files = [ + {file = "Pympler-1.1-py3-none-any.whl", hash = "sha256:5b223d6027d0619584116a0cbc28e8d2e378f7a79c1e5e024f9ff3b673c58506"}, + {file = "pympler-1.1.tar.gz", hash = "sha256:1eaa867cb8992c218430f1708fdaccda53df064144d1c5656b1e6f1ee6000424"}, +] + +[package.dependencies] +pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} + [[package]] name = "pymupdf" version = "1.26.3" @@ -4300,6 +4369,37 @@ files = [ {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, ] +[[package]] +name = "pywin32" +version = "311" +description = "Python for Window Extensions" +optional = false +python-versions = "*" +groups = ["dev"] +markers = "platform_system == \"Windows\"" +files = [ + {file = "pywin32-311-cp310-cp310-win32.whl", hash = "sha256:d03ff496d2a0cd4a5893504789d4a15399133fe82517455e78bad62efbb7f0a3"}, + {file = "pywin32-311-cp310-cp310-win_amd64.whl", hash = "sha256:797c2772017851984b97180b0bebe4b620bb86328e8a884bb626156295a63b3b"}, + {file = "pywin32-311-cp310-cp310-win_arm64.whl", hash = "sha256:0502d1facf1fed4839a9a51ccbcc63d952cf318f78ffc00a7e78528ac27d7a2b"}, + {file = "pywin32-311-cp311-cp311-win32.whl", hash = "sha256:184eb5e436dea364dcd3d2316d577d625c0351bf237c4e9a5fabbcfa5a58b151"}, + {file = "pywin32-311-cp311-cp311-win_amd64.whl", hash = "sha256:3ce80b34b22b17ccbd937a6e78e7225d80c52f5ab9940fe0506a1a16f3dab503"}, + {file = "pywin32-311-cp311-cp311-win_arm64.whl", hash = "sha256:a733f1388e1a842abb67ffa8e7aad0e70ac519e09b0f6a784e65a136ec7cefd2"}, + {file = "pywin32-311-cp312-cp312-win32.whl", hash = "sha256:750ec6e621af2b948540032557b10a2d43b0cee2ae9758c54154d711cc852d31"}, + {file = "pywin32-311-cp312-cp312-win_amd64.whl", hash = "sha256:b8c095edad5c211ff31c05223658e71bf7116daa0ecf3ad85f3201ea3190d067"}, + {file = "pywin32-311-cp312-cp312-win_arm64.whl", hash = "sha256:e286f46a9a39c4a18b319c28f59b61de793654af2f395c102b4f819e584b5852"}, + {file = "pywin32-311-cp313-cp313-win32.whl", hash = "sha256:f95ba5a847cba10dd8c4d8fefa9f2a6cf283b8b88ed6178fa8a6c1ab16054d0d"}, + {file = "pywin32-311-cp313-cp313-win_amd64.whl", hash = "sha256:718a38f7e5b058e76aee1c56ddd06908116d35147e133427e59a3983f703a20d"}, + {file = "pywin32-311-cp313-cp313-win_arm64.whl", hash = "sha256:7b4075d959648406202d92a2310cb990fea19b535c7f4a78d3f5e10b926eeb8a"}, + {file = "pywin32-311-cp314-cp314-win32.whl", hash = "sha256:b7a2c10b93f8986666d0c803ee19b5990885872a7de910fc460f9b0c2fbf92ee"}, + {file = "pywin32-311-cp314-cp314-win_amd64.whl", hash = "sha256:3aca44c046bd2ed8c90de9cb8427f581c479e594e99b5c0bb19b29c10fd6cb87"}, + {file = "pywin32-311-cp314-cp314-win_arm64.whl", hash = "sha256:a508e2d9025764a8270f93111a970e1d0fbfc33f4153b388bb649b7eec4f9b42"}, + {file = "pywin32-311-cp38-cp38-win32.whl", hash = "sha256:6c6f2969607b5023b0d9ce2541f8d2cbb01c4f46bc87456017cf63b73f1e2d8c"}, + {file = "pywin32-311-cp38-cp38-win_amd64.whl", hash = "sha256:c8015b09fb9a5e188f83b7b04de91ddca4658cee2ae6f3bc483f0b21a77ef6cd"}, + {file = "pywin32-311-cp39-cp39-win32.whl", hash = "sha256:aba8f82d551a942cb20d4a83413ccbac30790b50efb89a75e4f586ac0bb8056b"}, + {file = "pywin32-311-cp39-cp39-win_amd64.whl", hash = "sha256:e0c4cfb0621281fe40387df582097fd796e80430597cb9944f0ae70447bacd91"}, + {file = "pywin32-311-cp39-cp39-win_arm64.whl", hash = "sha256:62ea666235135fee79bb154e695f3ff67370afefd71bd7fea7512fc70ef31e3d"}, +] + [[package]] name = "pyyaml" version = "6.0.2" @@ -5622,4 +5722,4 @@ testing = ["coverage[toml]", "zope.event", "zope.testing"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "e3ad890e060c01ff399070b3ee67c913a8c2a31673461df967ad69a18859f4ef" +content-hash = "93667417a7b730c17d14c0c59149e2ab6386a4dfa7e663c1b7f2ebb804fbe4aa" @@ -87,6 +87,13 @@ djangorestframework-stubs = "^3.14.2" [tool.poetry.group.debug.dependencies] debugpy = "^1.8.1" + +[tool.poetry.group.dev.dependencies] +psutil = "^7.0.0" +objgraph = "^3.6.2" +pympler = "^1.1" +memory-profiler = "^0.61.0" + [tool.ruff] exclude = [ ".bzr",