@@ -8,5 +8,8 @@ "source.organizeImports": "explicit" }, "editor.wordBasedSuggestions": "currentDocument" - } + }, + "python-envs.defaultEnvManager": "ms-python.python:poetry", + "python-envs.defaultPackageManager": "ms-python.python:poetry", + "python-envs.pythonProjects": [] } @@ -0,0 +1,19 @@ +# Generated by Django 5.0.11 on 2025-08-15 22:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('authentication', '0019_alter_businessuserhost_allowed_models'), + ] + + operations = [ + migrations.AlterField( + model_name='customusermodel', + name='username', + field=models.CharField(default=None, max_length=100, unique=True, verbose_name='Username'), + preserve_default=False, + ), + ] @@ -135,9 +135,7 @@ class CustomUserModel(AbstractBaseUser, PermissionsMixin, BaseModel): username = models.CharField( max_length=100, unique=True, - null=True, - blank=True, - verbose_name=_('Username'), + verbose_name=_('Username') ) email = models.EmailField( max_length=100, @@ -231,6 +229,13 @@ class CustomUserModel(AbstractBaseUser, PermissionsMixin, BaseModel): except ObjectDoesNotExist: return None + @property + def host(self): + try: + return self.host_account + except ObjectDoesNotExist: + return None + def is_corporate(self): return self.account_type == 'business_host' @@ -1,9 +1,8 @@ +from django.core.exceptions import ObjectDoesNotExist + from authentication.models import ( - BusinessAccount, - BusinessUserHost, CustomUserModel, ) -from authentication.models.choices import AccountPrivileges class AccountStatusSelector: @@ -11,11 +10,13 @@ class AccountStatusSelector: self.user = user def is_business_host(self) -> bool: - return BusinessUserHost.objects.filter(user=self.user).exists() + return bool(self.user.host) def is_business_account(self) -> bool: - return BusinessAccount.objects.filter(user=self.user).exists() + try: + return bool(self.user.business_account) + except ObjectDoesNotExist: + return False def is_admin(self) -> bool: - accounts = BusinessAccount.objects.filter(user=self.user) - return not accounts.exists() or (accounts.first().account_privileges == AccountPrivileges.ADMIN) + return self.user.business_account.account_privileges == 'admin' if self.is_business_account() else False \ No newline at end of file @@ -11,13 +11,10 @@ class BusinessAccountSelector: @classmethod def from_user(cls, user: CustomUserModel, company: BusinessUserHost | None = None): - account = BusinessAccount.objects.filter(user=user) - if company is not None: - account.filter(parent_company=company) - if not account.exists(): + account = getattr(user, 'business_account', None) + if not account or (company is not None and account.parent_company != company): return None - - return cls(account.first()) + return cls(account) @classmethod def filter_by_email(cls, email: str, company: BusinessUserHost | None = None): @@ -99,11 +99,12 @@ class UserSelector: return 'regular' account = BusinessAccountSelector.from_user(self.user) - if account.account_type() == 'admin': + account_type = account.account_type() + if account_type == 'admin': return 'business_admin' - elif account.account_type() == 'regular': + elif account_type == 'regular': return 'business_account' - elif account.account_type() == 'sec': + elif account_type == 'sec': return 'business_security' def check_model_availability(self, model_title: str) -> bool: @@ -80,7 +80,7 @@ class BusinessAccountService: def accept(self): if self.account.acceptance_status == InvitationStatus.ACCEPTED: - raise Exception(_('Account is already confirmed')) + return self.update_status(InvitationStatus.ACCEPTED) def reject(self): @@ -1,56 +1,80 @@ +import jwt import logging from typing import Any from urllib.parse import parse_qs +from uuid import UUID -from asgiref.sync import async_to_sync, sync_to_async +from asgiref.sync import async_to_sync from channels.routing import URLRouter from channels.security.websocket import WebsocketDenier + from django.conf import settings from django.http import HttpRequest +from django.utils.translation import gettext as _ +from ninja.errors import HttpError from ninja.security import HttpBearer from oauth2_provider.models import AccessToken +from rest_framework import exceptions +from rest_framework.authentication import BaseAuthentication from authentication.exceptions import InvalidToken -from authentication.models.user import CustomUserModel +from authentication.models import CustomUserModel from authentication.services.token import TokenService logger = logging.getLogger(__name__) +class JWTAuthentication(BaseAuthentication): + def authenticate(self, request): + authorization_header = request.headers.get('Authorization') + if not authorization_header or not authorization_header.startswith('Bearer'): + return None + + try: + access_token = authorization_header.split(' ')[1] + payload = jwt.decode(access_token, settings.SECRET_KEY, algorithms=['HS256']) + except jwt.ExpiredSignatureError: + raise exceptions.AuthenticationFailed(_('Access token is expired')) + + user = CustomUserModel.objects.select_related( + 'payment_plan', + 'business_account', + 'business_account__group', + 'business_account__parent_company__user__payment_plan', + 'host_account', + ).get(uid=UUID(payload['uid'])) + + return (user, None) + + class SyncAuthBearer(HttpBearer): - def authenticate(self, _: HttpRequest, token: str) -> Any | None: + def authenticate(self, request: HttpRequest, token: str) -> Any | None: try: user_payload = async_to_sync(TokenService.decode)(token=token) return CustomUserModel.objects.get( **{key: user_payload[f'{key}'] for key in settings.JWT_SETTINGS['encode_attributes']} ) except InvalidToken: - ... - - try: - access = AccessToken.objects.prefetch_related('user').get(token=token) - return access.user - - except AccessToken.DoesNotExist: - return None + try: + access = AccessToken.objects.prefetch_related('user').get(token=token) + return access.user + except AccessToken.DoesNotExist: + raise HttpError(401, _('Access token expired or does not exist')) class AuthBearer(HttpBearer): - async def authenticate(self, _: HttpRequest, token: str) -> Any | None: + async def authenticate(self, request: HttpRequest, token: str) -> Any | None: try: user_payload = await TokenService.decode(token=token) return await CustomUserModel.objects.aget( **{key: user_payload[f'{key}'] for key in settings.JWT_SETTINGS['encode_attributes']} ) except InvalidToken: - ... - - try: - access = await AccessToken.objects.prefetch_related('user').aget(token=token) - return access.user - - except AccessToken.DoesNotExist: - return None + try: + access = await AccessToken.objects.prefetch_related('user').aget(token=token) + return access.user + except AccessToken.DoesNotExist: + raise HttpError(401, _('Access token expired or does not exist')) class WebsocketGlobalAuth: @@ -60,7 +84,7 @@ class WebsocketGlobalAuth: async def __call__(self, scope, receive, send): try: raw_token = parse_qs(scope['query_string']).get(b'token', ['...'])[0] - decoded_token = await sync_to_async(TokenService.decode)(token=raw_token) + decoded_token = await TokenService.decode(token=raw_token) scope['user'] = await CustomUserModel.objects.aget(id=decoded_token['id']) except Exception as exc: logger.exception(exc) @@ -0,0 +1,45 @@ +DICT_CONFIG = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'json': { + '()': 'backend.logging_formatters.JsonFormatter', + 'datefmt': '%Y-%m-%dT%H:%M:%S%z', + }, + }, + 'handlers': { + 'default': { + 'class': 'logging.StreamHandler', + 'formatter': 'json', + 'level': 'INFO', + 'stream': 'ext://sys.stdout', + }, + }, + 'loggers': { + 'django': { + 'handlers': ['default'], + 'level': 'INFO', + 'propagate': False, + }, + # 'uvicorn': { + # 'handlers': ['default'], + # 'level': 'INFO', + # 'propagate': False, + # }, + 'gunicorn': { + 'handlers': ['default'], + 'level': 'INFO', + 'propagate': False, + }, + '': { + 'handlers': ['default'], + 'level': 'INFO', + 'propagate': False, + }, + '__main__': { + 'handlers': ['default'], + 'level': 'INFO', + 'propagate': False, + }, + }, +} @@ -0,0 +1,21 @@ +import json +import logging +from logging import Formatter + + +class JsonFormatter(Formatter): + def format(self, record: logging.LogRecord) -> str: + log_record = { + 'timestamp': self.formatTime(record, self.datefmt), + 'message': record.getMessage(), + 'level': record.levelname, + } + if record.exc_info: + log_record.update( + { + 'exception': self.formatException(record.exc_info), + 'func_name': record.funcName, + 'lineno': record.lineno, + } + ) + return json.dumps(log_record, ensure_ascii=False) @@ -5,6 +5,9 @@ import pyroscope from celery.schedules import crontab from environs import Env +from backend.logging import DICT_CONFIG + +logging.config.dictConfig(DICT_CONFIG) env = Env() env.read_env() @@ -63,6 +66,7 @@ EXTERNAL_APPS = [ 'drf_spectacular_sidecar', 'ordered_model', 'import_export', + 'cacheops', ] @@ -70,7 +74,6 @@ INTERNAL_APPS = [ 'authentication.apps.AuthenticationConfig', 'ml_model.apps.MLModelConfig', 'messages.apps.MessagesConfig', - 'achievements.apps.AchievementsConfig', 'payments.apps.PaymentsConfig', 'reports.apps.ReportsConfig', 'stories.apps.StoriesConfig', @@ -81,9 +84,7 @@ INTERNAL_APPS = [ TOOLS = [ 'tools.apps.PublicAPIConfig', 'tools.apps.ChatsConfig', - 'tools.apps.CopywriteConfig', 'tools.apps.MediaConfig', - 'tools.apps.FeedConfig', ] @@ -107,14 +108,18 @@ AUTHENTICATION_BACKENDS = [ REST_FRAMEWORK = { 'DEFAULT_AUTHENTICATION_CLASSES': ( 'oauth2_provider.contrib.rest_framework.OAuth2Authentication', - 'rest_framework_simplejwt.authentication.JWTAuthentication', + 'authentication.security.JWTAuthentication', 'drf_social_oauth2.authentication.SocialAuthentication', ), 'EXCEPTION_HANDLER': 'core.utils.crutch_status_code_handler', + 'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',), 'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema', 'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.AllowAny',), + 'DEFAULT_RENDERER_CLASSES': ('rest_framework.renderers.JSONRenderer',), } + REST_USE_JWT = True + SIMPLE_JWT = { 'ACCESS_TOKEN_LIFETIME': env.timedelta('JWT_ACCESS_TOKEN_LIFETIME', 60 * 60 * 24), 'REFRESH_TOKEN_LIFETIME': env.timedelta('JWT_REFRESH_TOKEN_LIFETIME', 60 * 60 * 24), @@ -123,6 +128,7 @@ SIMPLE_JWT = { 'USER_ID_FIELD': 'uid', 'USER_ID_CLAIM': 'uid', } + SOCIAL_AUTH_ACTIVATE_JWT = True SOCIAL_AUTH_PIPELINE = ( @@ -299,8 +305,12 @@ CELERY_BEAT_SCHEDULE = { CACHES = { 'default': { - 'BACKEND': 'django.core.cache.backends.redis.RedisCache', - 'LOCATION': env.str('CACHE_URL', 'redis://cache-mdb:6379'), + 'BACKEND': 'django_redis.cache.RedisCache', + 'LOCATION': env.str('CACHE_BROKER_URL', 'redis://cache-mdb:6379'), + 'OPTIONS': { + 'CLIENT_CLASS': 'django_redis.client.DefaultClient', + }, + 'TIMEOUT': None, } } @@ -314,7 +324,7 @@ USE_I18N = True USE_TZ = True # Static files -STATIC_URL = 'static/' +STATIC_URL = 'djangostatic/' STATIC_ROOT = BASE_DIR / 'static' MEDIA_URL = 'media/' MEDIA_ROOT = BASE_DIR / 'static/media' @@ -492,3 +502,17 @@ if RELEASE and ENVIRONMENT and LOGGING_OTLP_SERVER: logger['handlers'].append('otlp') logging.config.dictConfig(LOGGING) + +CACHEOPS_REDIS = env.str('CACHEOPS_REDIS', CACHES['default']['LOCATION']) +CACHEOPS_DEGRADE_ON_FAILURE = True + +if CACHEOPS_REDIS: + CACHEOPS = { + 'authentication.*': {'ops': 'all', 'timeout': 60 * 60}, + 'ml_model.*': {'ops': 'all', 'timeout': 60 * 60}, + 'tools.chats.*': {'ops': 'all', 'timeout': 60 * 60}, + 'tools.media.*': {'ops': 'all', 'timeout': 60 * 60}, + 'payments.*': {'ops': 'all', 'timeout': 60 * 60}, + 'messages.*': {'ops': 'all', 'timeout': 60 * 60}, + 'reports.*': {'ops': 'all', 'timeout': 60 * 60}, + } \ No newline at end of file @@ -52,25 +52,23 @@ def healthz_status(request): urlpatterns = [ path('healthz/', healthz_status), - path('stories/', include('stories.urls')), - path('auth/', include('authentication.urls')), - path('payments/', include('payments.urls')), - path('djangoadmin/', admin.site.urls), - path('reports/', include('reports.urls')), - path('achievements/', include('achievements.urls')), - path('chats/', include('tools.chats.urls')), - path('feed/', include('tools.feed.urls')), - path('media/', include('tools.media.urls')), + path('api/v1/stories/', include('stories.urls')), + path('admin/', admin.site.urls), + path('api/v1/auth/', include('authentication.urls')), + path('api/v1/payments/', include('payments.urls')), + path('api/v1/reports/', include('reports.urls')), + path('api/v1/chats/', include('tools.chats.urls')), + path('api/v1/media/', include('tools.media.urls')), path( - 'schema-public/', + 'api/v1/schema-public/', SpectacularAPIView.as_view(urlconf=['backend.public']), name='schema-public', ), path( - 'public/', + 'api/v1/public/', SpectacularSwaggerView.as_view(url_name='schema-public'), ), - path('api/', api.urls), + path('api/v1/api/', api.urls), ] urlpatterns += public_urlpatterns @@ -78,9 +76,10 @@ urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) if settings.DEBUG: urlpatterns += [ - path('schema/', SpectacularAPIView.as_view(), name='schema'), + path('api/v1/schema/', SpectacularAPIView.as_view(), name='schema'), path( - 'schema/swagger-ui/', + 'api/v1/schema/swagger-ui/', SpectacularSwaggerView.as_view(url_name='schema'), ), ] + api.docs_url = '/docs' @@ -44,9 +44,9 @@ msgstr "Ярлык" msgid "Description" msgstr "Описание" -#: achievements/models.py:43 authentication/models/business_host.py:21 -#: authentication/models/email_token.py:12 authentication/models/user.py:247 -#: authentication/models/user.py:248 authentication/models/user_telegram.py:22 +#: achievements/models.py:43 authentication/models/business_host.py:22 +#: authentication/models/email_token.py:12 authentication/models/user.py:246 +#: authentication/models/user.py:247 authentication/models/user_telegram.py:22 #: authentication/models/user_vk.py:12 payments/models/invoice.py:15 #: payments/models/payment.py:26 payments/models/payment_plan.py:61 msgid "User" @@ -150,9 +150,11 @@ msgid "Group" msgstr "Группа" #: authentication/models/business_account.py:61 -msgid "Impossible to add this employee to this group which does not belong to this " +msgid "" +"Impossible to add this employee to this group which does not belong to this " "company" -msgstr "Невозможно добавить сотрудника к группе, когда он не принадлежит данной " +msgstr "" +"Невозможно добавить сотрудника к группе, когда он не принадлежит данной " "компании" #: authentication/models/business_account.py:70 @@ -498,7 +500,11 @@ msgstr "Аккаунт уже подтвержден" msgid "Passwords don't match" msgstr "Пароли не совпадают" -#: authentication/services/business_host_service.py:151 +#: authentication/services/business_account_service.py:111 +msgid "You cannot change the password of an unconfirmed e-mail user." +msgstr "Вы не можете изменить пароль неподтвержденного по e-mail пользователя." + +#: authentication/services/business_host_service.py:149 msgid "No user_email is provided" msgstr "" @@ -570,7 +576,7 @@ msgstr "Повторное приглашение сотруднику успе #: authentication/views.py:452 msgid "Could not confirm email, please try again." -msgstr "" +msgstr "Невозможно подтвердить email, попробуйте позже" #: backend/urls.py:30 msgid "Requested object does not exists" @@ -611,12 +617,50 @@ msgstr "Нейронные Модели" msgid "Inference is currently disabled, retry later." msgstr "Инференс в настоящее время выключен, повторите попытку позже." +#: ml_model/exceptions.py:17 +msgid "The model is currently disabled. Please try again later." +msgstr "" +"Модель в настоящее время неактивна. Пожалуйста, повторите попытку позже." + #: ml_model/exceptions.py:19 #, python-format msgid "Parameter %(parameter_name)s not valid, please retry later" msgstr "Параметр %(parameter_name)s некорректен, повторите попытку позже" +#: ml_model/exceptions.py:22 +msgid "The model is not responding" +msgstr "Модель не отвечает" + +#: ml_model/exceptions.py:31 +#, python-format +msgid "" +"The attached file format is not supported. Available formats: " +"%(available_extensions)s." +msgstr "" +"Формат вложенного файла не поддерживается. Доступные форматы: " +"%(available_extensions)s." + +#: ml_model/exceptions.py:37 +msgid "The length of the context has been exceeded." +msgstr "Длина контекста превышена." + +#: ml_model/exceptions.py:42 +msgid "Jinja template not found" +msgstr "Jinja-шаблон не найден" + +#: ml_model/exceptions.py:47 +msgid "There was an unknown error while rendering a template" +msgstr "При рендеринге шаблона произошла неизвестная ошибка" + +#: ml_model/models.py:28 ml_model/models.py:80 +msgid "Category" +msgstr "Категория" + #: ml_model/models.py:29 +msgid "Categories" +msgstr "Категории" + +#: ml_model/models.py:42 msgid "Not SVG-pictures not allowed" msgstr "Нельзя использовать не SVG-картинки" @@ -626,12 +670,45 @@ msgstr "Нельзя использовать не SVG-картинки" msgid "Tag" msgstr "Теги" -#: ml_model/models.py:40 ml_model/models.py:239 +#: ml_model/models.py:91 msgid "Tags" msgstr "Теги" -#: ml_model/models.py:45 ml_model/models.py:99 -#: reports/models/error_report.py:10 +#: ml_model/models.py:156 ml_model/models.py:403 +msgid "Model" +msgstr "Модель" + +#: ml_model/models.py:172 ml_model/models.py:173 +msgid "Settings" +msgstr "Настройки" + +#: ml_model/models.py:176 +#, python-format +msgid "Settings of %(model_title)s" +msgstr "Настройки %(model_title)s" + +#: ml_model/models.py:195 +#, python-format +msgid "%(model_title)s | %(version_name)s" +msgstr "%(model_title)s | %(version_name)s" + +#: ml_model/models.py:201 +msgid "Model Version" +msgstr "Версия Модели" + +#: ml_model/models.py:202 +msgid "Model Versions" +msgstr "Версии Модели" + +#: ml_model/models.py:211 +msgid "Versions" +msgstr "Версии" + +#: ml_model/models.py:212 +msgid "Link to versions" +msgstr "Привязка к версиям" + +#: ml_model/models.py:221 reports/models/error_report.py:10 msgid "Text" msgstr "Текст" @@ -647,10 +724,6 @@ msgstr "Эмбеддинги" msgid "ID" msgstr "ID" -#: ml_model/exceptions.py:20 -msgid "The model is not responding" -msgstr "Модель не отвечает" - #: ml_model/models.py:63 msgid "Runner" msgstr "Раннер" @@ -676,7 +749,7 @@ msgstr "Деплоймент" msgid "Deployments" msgstr "Деплойменты" -#: ml_model/models.py:100 ml_model/models.py:346 stories/models.py:36 +#: ml_model/models.py:222 stories/models.py:36 msgid "Image" msgstr "Картинка" @@ -718,6 +791,11 @@ msgstr "Обязательный" msgid "%(input_type)s input of %(deployment_title)s" msgstr "Входящий поток типа %(input_type)s деплоймента %(deployment_title)s" +#: ml_model/models.py:124 +#, python-format +msgid "%(model_title)s | %(input_type)s" +msgstr "%(model_title)s | %(input_type)s" + #: ml_model/models.py:130 #, fuzzy #| msgid "Model Input" @@ -837,11 +915,23 @@ msgid "Payment Rule \"%(strategy)s\"/\"%(interaction_type)s\" of " "%(deployment_title)s" msgstr "Платежное правило \"%(strategy)s\"/\"%(interaction_type)s\" деплоймента %(deployment_title)s" -#: ml_model/models.py:222 +#: ml_model/models.py:327 +msgid "Coefficient" +msgstr "Коэффициент" + +#: ml_model/models.py:328 +msgid "Cost multiplier" +msgstr "Цена" + +#: ml_model/models.py:335 +msgid "Rate" +msgstr "Ставка" + +#: ml_model/models.py:339 msgid "Payment Rule" msgstr "Платежное правило" -#: ml_model/models.py:223 +#: ml_model/models.py:340 msgid "Payment Rules" msgstr "Платежные правила" @@ -877,10 +967,6 @@ msgstr "Сложение" msgid "Multiplication" msgstr "Умножение" -#: ml_model/models.py:295 -msgid "Coefficient" -msgstr "Коэффициент" - #: ml_model/models.py:308 #, python-format msgid "Payment Bias of %(inference_title)s" @@ -931,7 +1017,24 @@ msgstr "Аватар" msgid "Neuron Model" msgstr "Нейронная Модель" -#: ml_model/selectors/ml_models_selector.py:57 +#: ml_model/models.py:401 +msgid "Descriptor" +msgstr "Дескриптор" + +#: ml_model/models.py:407 +#, python-format +msgid "Instruction of %(model_title)s" +msgstr "Инструкция %(model_title)s" + +#: ml_model/models.py:410 +msgid "Model Instruction" +msgstr "Инструкция Модели" + +#: ml_model/models.py:411 +msgid "Model Instructions" +msgstr "Инструкции Моделей" + +#: ml_model/selectors/ml_models_selector.py:81 msgid "no model by this id" msgstr "Не найдено моделей по этому ID" @@ -1269,60 +1372,20 @@ msgstr "" "Случилась ошибка во время генерации. Она может возникать из-за того, что " "NSFW-контент запрещен. Попробуйте снова" -#~ msgid "Category" -#~ msgstr "Категория" - -#~ msgid "Categories" -#~ msgstr "Категории" - #~ msgid "Model Tag" #~ msgstr "Тег модели" #~ msgid "Model Tags" #~ msgstr "Теги модели" -#~ msgid "Model" -#~ msgstr "Модель" - -#~ msgid "Settings" -#~ msgstr "Настройки" - -#, python-format -#~ msgid "Settings of %(model_title)s" -#~ msgstr "Настройки %(model_title)s" - -#, python-format -#~ msgid "%(model_title)s | %(version_name)s" -#~ msgstr "%(model_title)s | %(version_name)s" - -#~ msgid "Model Version" -#~ msgstr "Версия Модели" - -#~ msgid "Model Versions" -#~ msgstr "Версии Модели" - -#~ msgid "Versions" -#~ msgstr "Версии" - -#~ msgid "Link to versions" -#~ msgstr "Привязка к версиям" - -#, python-format -#~ msgid "%(model_title)s | %(input_type)s" -#~ msgstr "%(model_title)s | %(input_type)s" - #~ msgid "List" #~ msgstr "Список" -#~ msgid "Cost multiplier" -#~ msgstr "Цена" - -#~ msgid "Rate" -#~ msgstr "Ставка" - - -msgid "You cannot change the password of an unconfirmed e-mail user." -msgstr "Вы не можете изменить пароль неподтвержденного по e-mail пользователя." - msgid "Unknown file format" msgstr "Неизвестный формат файла" + +msgid "Access token is expired" +msgstr "Срок действия токена доступа истек" + +msgid "Token prefix is missing" +msgstr "Отсутствует префикс токена" @@ -0,0 +1,31 @@ +# Generated by Django 5.0.11 on 2025-08-15 22:09 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('msgs', '0004_alter_message_info'), + ] + + operations = [ + migrations.AlterField( + model_name='message', + name='content_type', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.DO_NOTHING, to='contenttypes.contenttype'), + preserve_default=False, + ), + migrations.AlterField( + model_name='message', + name='object_id', + field=models.UUIDField(default=None), + preserve_default=False, + ), + migrations.AddIndex( + model_name='message', + index=models.Index(fields=['content_type', 'object_id'], name='msgs_messag_content_5cea8d_idx'), + ), + ] @@ -53,8 +53,8 @@ class Message(models.Model): verbose_name='Избранное', ) is_shared = models.BooleanField(default=False, verbose_name='Сообщение в фиде') - content_type = models.ForeignKey(ContentType, blank=True, null=True, on_delete=models.DO_NOTHING) - object_id = models.UUIDField(blank=True, null=True) + content_type = models.ForeignKey(ContentType, on_delete=models.DO_NOTHING) + object_id = models.UUIDField() content_object = fields.GenericForeignKey('content_type', 'object_id') info = models.JSONField( verbose_name='Мета-информация', @@ -70,3 +70,4 @@ class Message(models.Model): verbose_name = 'Сообщение' verbose_name_plural = 'Сообщения' ordering = ['-created_at'] + indexes = [models.Index(fields=['content_type', 'object_id'])] @@ -1,4 +1,10 @@ +from typing import Dict, Any + +from backend import settings + +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers +from rest_framework.serializers import ValidationError from messages.models import Message @@ -31,3 +37,22 @@ class MessageSerializer(serializers.ModelSerializer): 'is_sent', 'info', ] + + def validate(self, data: Dict[str, Any]) -> Dict[str, Any]: + file = data.get('file') + version = data.get('info', {}).get('inference', 'default') + max_mb_size = settings.MAX_UPLOAD_SIZE_PER_MODEL.get( + version, + settings.MAX_UPLOAD_SIZE_PER_MODEL['default'] + ) + if file and file.size > (max_mb_size << 10 << 10): + raise ValidationError( + _('The file size cannot exceed %(max_mb_size)d MB') % {'max_mb_size': max_mb_size} + ) + return data + + def to_representation(self, instance): + ret = super().to_representation(instance) + if 'content' in ret and ret['content']: + ret['content'] = ret['content'].replace('\\n', '\n') + return ret @@ -1,6 +1,6 @@ from ml_model.runners.dummy import DummyImageRunner, DummyTextRunner from ml_model.runners.falai import FalAIRunner -from ml_model.runners.openai import OpenAIGPTRunner +from ml_model.runners.openai import OpenAIGPTRunner, OpenAIResponseRunner, GPTImageRunner from ml_model.runners.openrouter import OpenrouterRunner from ml_model.runners.replicate import ( ReplicateAudioRunner, @@ -11,6 +11,8 @@ from ml_model.runners.replicate import ( __all__ = [ 'OpenAIGPTRunner', + 'OpenAIResponseRunner', + 'GPTImageRunner', 'OpenrouterRunner', 'ReplicateTextRunner', 'ReplicateAudioRunner', @@ -1,7 +1,6 @@ import base64 import json import logging -import re import uuid from abc import ABC, abstractmethod from io import BytesIO, StringIO @@ -77,9 +76,9 @@ class OpenAICompatibleRunner(BaseRunner, ABC): ) elif file and isinstance(file, StringIO): file_content = file.getvalue() - if len(file_data := file.getvalue()) > 20_000: + if len(file_content) > 20_000: chunks = TextSplitterTool().split_text( - text=file_data, separators=["\n\n", "\n", ".", " ", ""] + text=file_content, separators=["\n\n", "\n", ".", " ", ""] ) for proxy in proxies: try: @@ -166,3 +165,185 @@ class OpenAIGPTRunner(OpenAICompatibleRunner): raise ParameterNotValid('model') elif error['code'] == 'invalid_value': raise ParameterNotValid(error['param']) + elif error['code'] == 'invalid_type': + raise ParameterNotValid(error['param']) + elif error['code'] == 'unknown_parameter': + raise ParameterNotValid(error['param']) + + +class OpenAIResponseRunner(OpenAIGPTRunner): + @classmethod + def generate(cls, content=None, file=None, parameters={}, history=[], scrape_results=[]): + proxies = Proxy.objects.all() + payload = { + 'input': [ + *[ + { + 'role': 'user' if not message.from_model else 'assistant', + 'content': stripped_message, + } + for message in history + if message.content and (stripped_message := message.content.strip()) + ], + ], + 'stream': True, + 'instructions': 'Форматирование — обязательное требование. Выполняй строго по правилам:\\n\\n1) ' + 'Используй реальные символы новой строки. Не выводи "\\\\n" как текст — вставляй ' + 'переносы (символ новой строки).\\n2) Между абзацами ставь ОДНУ пустую строку ' + '(то есть два символа новой строки подряд: \\\\n\\\\n).\\n3) Для списков — каждый пункт на ' + 'отдельной строке; между списком и текстом — пустая строка.\\n4) ' + 'Любые блоки/куски/фрагменты кода СТРОГО ' + 'в тройных бэктиках (```) с указанием наименования языка программирования, ' + 'с пустой строкой перед и после блока/куска/фрагмента кода.' + '5) Не используй HTML.\\n6) Если формат неверный — перепиши ответ и ' + 'верни исправленный вариант.', + **parameters, + } + for result in scrape_results: + if isinstance(result, StringIO): + payload['input'].append( + {'role': 'user', 'content': [{'type': 'input_text', 'text': result.getvalue()}]} + ) + + payload['input'].append({'role': 'user', 'content': [{'type': 'input_text', 'text': content}]}) + + if file and isinstance(file, BytesIO): + mime = filetype.guess(file.read(20)).mime + file.seek(0) + payload['input'][-1]['content'].append( + { + 'type': 'input_image', + 'image_url': f'data:{mime};base64,{base64.b64encode(file.getvalue()).decode("utf-8")}' + } + ) + elif file and isinstance(file, StringIO): + file_content = file.getvalue() + if len(file_data := file.getvalue()) > 20_000: + chunks = TextSplitterTool().split_text( + text=file_data, separators=["\n\n", "\n", ".", " ", ""] + ) + for proxy in proxies: + try: + file_content = EmbeddingTool( + settings.REDIS_HOST, + settings.REDIS_PORT, + proxy.protocol, + proxy.address, + cls.AUTHORIZATION_TOKEN, + settings.MAX_THREADS + ).convert( + document_name=chunks[0].partition(f':{chr(10)}')[2].split(f'{chr(10)}')[0][:100], + chunks=chunks, + file_uid=str(uuid.uuid4()).replace('-', '_'), + user_prompt=content + ) + payload['input'].pop() + except httpx.ConnectError: + continue + except httpx.TimeoutException as exc: + logger.exception(exc) + raise Exception('Timeout happened') + payload['input'].append( + {'role': 'user', 'content': [{'type': 'input_text', 'text': file_content}]} + ) + for proxy in proxies: + with httpx.Client( + base_url=cls.BASE_URL, + headers={ + 'Authorization': f'Bearer {cls.AUTHORIZATION_TOKEN}', + 'Content-Type': 'application/json', + }, + proxy=f'{proxy.protocol}://{proxy.address}', + timeout=600, + ) as client: + try: + with client.stream('POST', '/responses', json=payload) as stream: + stream_content = stream.iter_lines() + if stream.status_code >= 400: + raw = ''.join([chunk for chunk in stream_content]) + try: + errors: dict[Literal['error'], dict[Literal['message'] | str, Any]] = ( + json.loads(raw).get('error', {}) + ) + except json.decoder.JSONDecodeError: + logger.error(raw) + errors = raw + cls.map_errors(errors) + for chunk in stream_content: + if chunk.strip() in cls.SKIP_TOKENS: + continue + if chunk.strip() in cls.END_TOKENS: + break + try: + data_str = chunk[5:].strip() + dict_ = json.loads(data_str) + if dict_.get("type") == "response.output_text.delta": + text = dict_.get("delta", "") + if text: + yield text + except json.decoder.JSONDecodeError: + continue + except httpx.ConnectError: + continue + except httpx.TimeoutException as exc: + logger.exception(exc) + raise Exception('Timeout happened') + + +class GPTImageRunner(OpenAIGPTRunner): + @classmethod + def generate(cls, content=None, file=None, parameters={}, history=[], scrape_results=[]): + payload = { + 'prompt': content, + 'stream': True, + **parameters + } + + if file and isinstance(file, BytesIO): + files = [ + ('image[]', ('image.png', file, 'image/png')) + ] + + for proxy in Proxy.objects.all(): + with httpx.Client( + base_url=cls.BASE_URL, + headers={ + 'Authorization': f'Bearer {cls.AUTHORIZATION_TOKEN}', + }, + proxy=f'{proxy.protocol}://{proxy.address}', + timeout=600, + ) as client: + if file and isinstance(file, BytesIO): + streaming = client.stream('POST', 'images/edits', data=payload, files=files) + else: + streaming = client.stream('POST', 'images/generations', json=payload) + try: + with streaming as stream: + stream_content = stream.iter_lines() + if stream.status_code >= 400: + raw = ''.join([chunk for chunk in stream_content]) + try: + errors: dict[Literal['error'], dict[Literal['message'] | str, Any]] = ( + json.loads(raw).get('error', {}) + ) + except json.decoder.JSONDecodeError: + logger.error(raw) + errors = raw + cls.map_errors(errors) + for chunk in stream_content: + if chunk.strip() in cls.SKIP_TOKENS: + continue + if chunk.strip() in cls.END_TOKENS: + break + try: + dict_ = json.loads(chunk[5:].strip()) + if dict_.get("type") in ("image_edit.completed", "image_generation.completed"): + if b64_json := dict_.get("b64_json", ""): + yield b64_json + except json.decoder.JSONDecodeError: + continue + except httpx.ConnectError: + continue + except httpx.TimeoutException as exc: + logger.exception(exc) + raise Exception('Timeout happened') @@ -26,7 +26,8 @@ from messages.models import Message from ml_model.exceptions import ( InferenceDisabled, PaymentRuleNotImplemented, - ScraperDoesNotExists, UnknownFileException, + ScraperDoesNotExists, + UnknownFileException, ) from ml_model.models import ( Deployment, @@ -265,7 +266,7 @@ class InferenceService: ): if isinstance(file, StringIO) and len(file_data := file.getvalue()) >= 20_000: calculated_price += TokenizerTool.token_count( - file_data, biggest_coefficient=0.15 + file_data ) * payment_rule.cost elif ( content @@ -360,7 +361,6 @@ class InferenceService: process_time = timedelta(seconds=end - start) output_content = output_content.split('base64,')[-1] # cutoff b64-prefix if exists - output_slot.elapsed_time = process_time match inference.deployment.output_type: case Deployment.OutputTypeChoices.TEXT: @@ -379,7 +379,6 @@ class InferenceService: and payment_rule.interaction_type == PaymentRule.InteractionTypeChoices.OUTPUT ): calculated_price += TokenizerTool.token_count(output_content) * payment_rule.cost - for payment_bias in inference.payment_biases: if payment_bias.type == PaymentBias.TypeChoices.ADDITION: calculated_price += payment_bias.coefficient @@ -8,7 +8,7 @@ from typing import List from redis.commands.search.document import Document from redis.commands.search.query import Query -from .tasks import drop_redis_vectors +from ml_model.tools.tasks import drop_redis_vectors class EmbeddingTool: @@ -1,4 +1,4 @@ -import unicodedata +import tiktoken class TokenizerTool: @@ -8,35 +8,8 @@ class TokenizerTool: def token_count( self, text: str, - subword_step: int = 3, - smaller_coefficient: float = 0.15, - biggest_coefficient: float = 0.05, - token_bias: int = 0 ) -> int: - if not text: - return 0 - text = unicodedata.normalize("NFC", text) - count = 0 - i = 0 - while i < len(text): - c = text[i] - if c.isspace(): - count += 1 - i += 1 - elif c in '.,!?;:()[]{}"\'«»—–-0123456789': - count += 1 - i += 1 - else: - utf_bytes = c.encode('utf-8') - j = i + 1 - while j < len(text): - next_char = text[j] - if next_char.isspace() or next_char in '.,!?;:()[]{}"\'«»—–-0123456789': - break - utf_bytes += next_char.encode('utf-8') - j += 1 - count += max(1, len(utf_bytes) // subword_step) - i = j - if count < 100: - return int(round(count + count * smaller_coefficient) + token_bias) - return int(round(count / 2 + count * biggest_coefficient) + token_bias) \ No newline at end of file + encoding = tiktoken.get_encoding('o200k_base') + num_tokens = len(encoding.encode(text)) + return int(round(num_tokens + num_tokens * 0.20)) + @@ -12,7 +12,7 @@ class APIKeySelector(BaseSelector): return api_keys def get_by_name(self, name: str, serialize: bool = False): - api_key = APIKey.objects.get(user=self.user, name=name) + api_key = APIKey.objects.get(user=self.user, name=name, is_deleted=False) if serialize: return APIKeyResultSerializer(api_key) return api_key @@ -1,13 +1,16 @@ # CORE SETTINGS SECRET_KEY=testtest +DEBUG=true TRACE_ID_HEADER=X-Trace-ID SESSION_ID_HEADER=X-Session-ID # NEURON MODELS OPENAI_API_KEY=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 -REPLICATE_API_KEY=r8_HBk6Ts5UJU60nDOUl1V6Uej4ihAAxUc3HAZLO -OPENROUTER_API_KEY=r8_HBk6Ts5UJU60nDOUl1V6Uej4ihAAxUc3HAZLO +REPLICATE_API_KEY=r8_4IjhLLMyKyq3nm8qauTndtdOMixxmep3uRQMu +OPENROUTER_API_KEY=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 GOOGLE_API_KEY=AIzaSyBf9el4d_CY610zjCcesKxKL70BLfl57OM +FAL_API_KEY=617f0fe4-c627-4119-9681-11af2c3e416a:3d618ccd0ee11ed82543801d7da96d1d +SERPER_API_KEY=ed8e0dbcc26dacf3f7f99fbc8b3add9ada0c793e # EXTERNAL SERVICES OPENAI_PROXY_HOST=neuron-proxy:8080 @@ -717,6 +717,23 @@ tzdata = {version = "*", markers = "sys_platform == \"win32\""} argon2 = ["argon2-cffi (>=19.1.0)"] bcrypt = ["bcrypt"] +[[package]] +name = "django-cacheops" +version = "7.2" +description = "A slick ORM cache with automatic granular event-driven invalidation for Django." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "django_cacheops-7.2-py2.py3-none-any.whl", hash = "sha256:c3b4399474919e62aa91bbd97e7b68cfc50e8cb4384d743bde686c37fe8c010b"}, + {file = "django_cacheops-7.2.tar.gz", hash = "sha256:cbc11cc0321295a3644e27bcb26940f08cb9c2e71d3ee506cbcff0bdda1a9f33"}, +] + +[package.dependencies] +django = ">=3.2" +funcy = ">=1.8" +redis = ">=3.0.0" + [[package]] name = "django-celery-beat" version = "2.8.1" @@ -753,6 +770,21 @@ files = [ asgiref = ">=3.6" django = ">=4.2" +[[package]] +name = "django-filter" +version = "25.1" +description = "Django-filter is a reusable Django application for allowing users to filter querysets dynamically." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "django_filter-25.1-py3-none-any.whl", hash = "sha256:4fa48677cf5857b9b1347fed23e355ea792464e0fe07244d1fdfb8a806215b80"}, + {file = "django_filter-25.1.tar.gz", hash = "sha256:1ec9eef48fa8da1c0ac9b411744b16c3f4c31176c867886e4c48da369c407153"}, +] + +[package.dependencies] +Django = ">=4.2" + [[package]] name = "django-import-export" version = "4.3.9" @@ -864,6 +896,25 @@ files = [ [package.dependencies] Django = ">=2.1" +[[package]] +name = "django-redis" +version = "6.0.0" +description = "Full featured redis cache backend for Django." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "django_redis-6.0.0-py3-none-any.whl", hash = "sha256:20bf0063a8abee567eb5f77f375143c32810c8700c0674ced34737f8de4e36c0"}, + {file = "django_redis-6.0.0.tar.gz", hash = "sha256:2d9cb12a20424a4c4dde082c6122f486628bae2d9c2bee4c0126a4de7fda00dd"}, +] + +[package.dependencies] +Django = ">=4.2" +redis = ">=4.0.2" + +[package.extras] +hiredis = ["redis[hiredis] (>=4.0.2)"] + [[package]] name = "django-timezone-field" version = "7.1" @@ -1074,6 +1125,18 @@ files = [ {file = "filetype-1.2.0.tar.gz", hash = "sha256:66b56cd6474bf41d8c54660347d37afcc3f7d1970648de365c102ef77548aadb"}, ] +[[package]] +name = "funcy" +version = "2.0" +description = "A fancy and practical functional tools" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "funcy-2.0-py2.py3-none-any.whl", hash = "sha256:53df23c8bb1651b12f095df764bfb057935d49537a56de211b098f4c79614bb0"}, + {file = "funcy-2.0.tar.gz", hash = "sha256:3963315d59d41c6f30c04bc910e10ab50a3ac4a225868bfa96feed133df075cb"}, +] + [[package]] name = "googleapis-common-protos" version = "1.70.0" @@ -3633,4 +3696,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "8873cb6bd8e2cfa1ec8c5c259861dd80197643011f2a8668386202e3eeabec40" +content-hash = "02e81143df2dc0aadc8dd1ab3ba9ac6e14f03d302e44d35c1ec7c8483cbcbc2a" @@ -51,6 +51,9 @@ orjson = "^3.11.0" opentelemetry-sdk = "^1.36.0" opentelemetry-exporter-otlp = "^1.36.0" numpy = "^2.3.2" +django-cacheops = "^7.2" +django-filter = "^25.1" +django-redis = "^6.0.0" [tool.poetry.group.dev.dependencies]