@@ -1,21 +0,0 @@ -{ - "version": "0.0.1", - "configurations": [ - { - "name": "Python Debugger", - "type": "debugpy", - "request": "attach", - "connect": { - "port": 5678 - }, - "justMyCode": true, - "pathMappings": [ - { - "localRoot": "${workspaceFolder}", - "remoteRoot": "/code" - } - ], - "django": true, - } - ] -} @@ -1,15 +0,0 @@ -{ - "files.eol": "\n", - "[python]": { - "editor.formatOnSave": true, - "editor.defaultFormatter": "charliermarsh.ruff", - "editor.codeActionsOnSave": { - "source.fixAll": "explicit", - "source.organizeImports": "explicit" - }, - "editor.wordBasedSuggestions": "currentDocument" - }, - "python-envs.defaultEnvManager": "ms-python.python:poetry", - "python-envs.defaultPackageManager": "ms-python.python:poetry", - "python-envs.pythonProjects": [] -} @@ -62,7 +62,7 @@ class UserService: user.set_password(user_data['password']) else: user = CustomUserModel.objects.create_user( - email=email, password=user_data['password'] + email=email, password=user_data['password'], is_subscribed_to_emails=user_data.get('is_subscribed_to_emails', False) ) UTMService(user).create_utm( user_data.get('utm_source'), @@ -20,6 +20,7 @@ from payments.serializers import PromoCodeSerializer, UserPlanDetailSerializer class NewUserSerializer(serializers.Serializer): email = serializers.EmailField() password = serializers.CharField() + is_subscribed_to_emails = serializers.BooleanField(required=False, default=False) referer = serializers.CharField(required=False) utm_source = serializers.CharField(required=False, allow_blank=True, max_length=255) utm_medium = serializers.CharField(required=False, allow_blank=True, max_length=255) @@ -1,18 +1,7 @@ import os -from channels.routing import ProtocolTypeRouter, URLRouter from django.core.asgi import get_asgi_application -from django.urls import path os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings') application = get_asgi_application() - -from tools.copywrite.routes.v1 import ws_router as copywrite_ws_router # noqa: E402 - -application = ProtocolTypeRouter( - { - 'http': application, - 'websocket': URLRouter([path('rtc/copywrite/', copywrite_ws_router)]), - } -) @@ -54,13 +54,13 @@ EXTERNAL_APPS = [ 'drf_social_oauth2', 'dj_rest_auth', 'dj_rest_auth.registration', - 'django_celery_beat', 'django_minio_backend', 'drf_spectacular', 'drf_spectacular_sidecar', 'ordered_model', 'import_export', 'cacheops', + 'django_celery_beat', ] @@ -105,7 +105,6 @@ REST_FRAMEWORK = { 'authentication.security.JWTAuthentication', 'drf_social_oauth2.authentication.SocialAuthentication', ), - 'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',), 'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema', 'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.AllowAny',), 'DEFAULT_RENDERER_CLASSES': ('rest_framework.renderers.JSONRenderer',), @@ -189,22 +188,6 @@ TEMPLATES = [ WSGI_APPLICATION = 'backend.wsgi.application' ASGI_APPLICATION = 'backend.asgi.application' -CHANNEL_LAYERS = { - 'default': { - 'BACKEND': 'channels_redis.core.RedisChannelLayer', - 'CONFIG': { - 'hosts': [ - ( - env.str('CHANNELS_MDB_HOST', 'channels-mdb'), - env.int('CHANNELS_MDB_PORT', 6379), - ) - ], - 'capacity': 1500, - 'expiry': 20, - }, - }, -} - # Database DATABASES = { 'default': { @@ -228,45 +211,44 @@ JWT_SETTINGS = { DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' - STORAGES = { 'default': { 'BACKEND': 'django_minio_backend.models.MinioBackend', + 'OPTIONS': { + 'MINIO_ENDPOINT': env.str('MINIO_ENDPOINT', default='s3:9000'), + 'MINIO_USE_HTTPS': env.bool('MINIO_USE_HTTPS', default=False), + 'MINIO_EXTERNAL_ENDPOINT': env.str('MINIO_EXTERNAL_ENDPOINT', default='localhost:9000'), + 'MINIO_EXTERNAL_ENDPOINT_USE_HTTPS': env.bool( + 'MINIO_EXTERNAL_ENDPOINT_USE_HTTPS', default=False + ), + 'MINIO_ACCESS_KEY': env.str('MINIO_ACCESS_KEY'), + 'MINIO_SECRET_KEY': env.str('MINIO_SECRET_KEY'), + 'MINIO_PRIVATE_BUCKETS': [ + 'air-messages', + 'air-media', + 'air-achievements', + 'air-templates-pictures', + 'air-stories', + 'air-profiles', + 'air-models', + 'air-media-presets', + 'air-voices', + ], + 'MINIO_PUBLIC_BUCKETS': [], + 'MINIO_DEFAULT_BUCKET': 'air-media', + 'MINIO_CONSISTENCY_CHECK_ON_START': False, + 'MINIO_BUCKET_CHECK_ON_SAVE': False, + 'MINIO_MULTIPART_UPLOAD': False, + 'MINIO_MULTIPART_THRESHOLD': 6 * 1024 * 1024, + 'MINIO_MULTIPART_PART_SIZE': 6 * 1024 * 1024, + 'MINIO_URL_CACHING_ENABLED': True, + 'MINIO_URL_CACHE_TIMEOUT': 60 * 60 * 8, + 'MINIO_URL_CACHE_PREFIX': 'minio_url_', + }, }, - 'import_export': { - 'BACKEND': 'django_minio_backend.models.MinioBackend', - }, - 'staticfiles': { - 'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage', - }, + 'staticfiles': {'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage'}, } -MINIO_BUCKET_CHECK_ON_SAVE = env.bool('MINIO_BUCKET_CHECK_ON_SAVE', default=False) -MINIO_URL = env.str('MINIO_URL', default='localhost') -MINIO_PORT = env.str('MINIO_PORT', default='9000') -MINIO_ENDPOINT = env.str('MINIO_ENDPOINT') -MINIO_USE_HTTPS = env.bool('MINIO_USE_HTTPS', default=False) -MINIO_EXTERNAL_ENDPOINT = env.str('MINIO_EXTERNAL_ENDPOINT', default='localhost:9000') -MINIO_EXTERNAL_ENDPOINT_USE_HTTPS = env.bool('MINIO_EXTERNAL_ENDPOINT_USE_HTTPS', default=False) - -MINIO_ACCESS_KEY = env.str('MINIO_ACCESS_KEY') -MINIO_SECRET_KEY = env.str('MINIO_SECRET_KEY') -MINIO_PRIVATE_BUCKETS = [ - 'air-messages', - 'air-achievements', - 'air-templates-pictures', - 'air-stories', - 'air-profiles', - 'air-models', - 'air-media-presets', - 'air-voices', -] -MINIO_STATIC_FILES_BUCKET = 'air-static' -MINIO_PRIVATE_BUCKETS.append(MINIO_STATIC_FILES_BUCKET) - -MINIO_MEDIA_FILES_BUCKET = 'air-media' -MINIO_PRIVATE_BUCKETS.append(MINIO_MEDIA_FILES_BUCKET) - SPECTACULAR_SETTINGS = { 'TITLE': 'AIR', 'DESCRIPTION': 'AIR Project API conf', @@ -20,7 +20,6 @@ api = NinjaAPI(title='AIR API', version='1.0.0', docs_url=None) compatibility_api = NinjaAPI(title='AIR API DEBUG', version='0.0.1', docs_url=None) compatibility_api_v2 = NinjaAPI(title='AIR API DEBUG v2', version='2.0.0', docs_url=None) -api.add_router('copywrite/', 'tools.copywrite.routes.v1.router') api.add_router('users/', 'users.routes.v1.router') api.add_router('chats/', 'tools.chats.routes.v1.router') api.add_router('media/', 'tools.media.routes.v1.router') @@ -1,10 +1,8 @@ import redis - -from django.core.management.base import BaseCommand - from django.conf import settings +from django.core.management.base import BaseCommand from redis.commands.search.field import TagField, TextField, VectorField -from redis.commands.search.indexDefinition import IndexDefinition, IndexType +from redis.commands.search.index_definition import IndexDefinition, IndexType class Command(BaseCommand): @@ -1,10 +1,9 @@ +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import List, Tuple + import httpx import numpy as np import redis - -from concurrent.futures import ThreadPoolExecutor, as_completed -from typing import Tuple, List - from django.conf import settings from langchain_text_splitters import RecursiveCharacterTextSplitter from pydantic.v1 import UUID4 @@ -155,4 +154,4 @@ class EmbeddingService: ] drop_redis_vectors.delay(message_uid) redis_client.close() - return embedding_tokens, result \ No newline at end of file + return embedding_tokens, result @@ -26,10 +26,10 @@ class MinIOService: def __init__(self): self.minio = Minio( - endpoint=f'{settings.MINIO_URL}:{settings.MINIO_PORT}', - access_key=settings.MINIO_ACCESS_KEY, - secret_key=settings.MINIO_SECRET_KEY, - secure=settings.MINIO_USE_HTTPS, + endpoint=settings.STORAGES['default']['OPTIONS']['MINIO_ENDPOINT'], + access_key=settings.STORAGES['default']['OPTIONS']['MINIO_ACCESS_KEY'], + secret_key=settings.STORAGES['default']['OPTIONS']['MINIO_SECRET_KEY'], + secure=settings.STORAGES['default']['OPTIONS']['MINIO_USE_HTTPS'], ) def put_object(self, obj: BytesIO, filename: str, dest: str) -> str: @@ -1,33 +1,13 @@ -from typing import List, Optional, Any +from typing import Any + from ninja import ModelSchema, Schema from pydantic import condecimal from ml_model.models import ( - ConfigurationParameter, - ModelConfiguration, NeuronModel, ) -class ConfigurationParameterSchema(ModelSchema): - class Meta: - model = ConfigurationParameter - exclude = ('configuration',) - - -class ModelConfigurationSchema(ModelSchema): - parameters: List[ConfigurationParameterSchema] = [] - model_id: str - version_id: Optional[str] = None - - class Meta: - model = ModelConfiguration - exclude = ('ct', 'oid', 'obj', 'model') - - class Config: - protected_namespaces = () - - class NeuronModelLink(ModelSchema): class Meta: model = NeuronModel @@ -42,4 +22,4 @@ class PredictPriceInputSchema(Schema): class PredictPriceSchema(Schema): - price: condecimal(max_digits=10, decimal_places=2) | None \ No newline at end of file + price: condecimal(max_digits=10, decimal_places=2) | None @@ -1,12 +1,7 @@ -import redis -import tiktoken - from random import randint -from typing import Literal, List, Dict, Any, Tuple +from typing import Any, Dict, List, Literal, Tuple -from django.conf import settings -from redis.commands.search.field import TagField, TextField, VectorField -from redis.commands.search.indexDefinition import IndexDefinition, IndexType +import tiktoken from authentication.models import CustomUserModel from authentication.selectors.account_status_selector import ( @@ -19,12 +14,12 @@ from authentication.selectors.business_account_selector import ( def random_with_N_digits(n): range_start = 10 ** (n - 1) - range_end = (10 ** n) - 1 + range_end = (10**n) - 1 return randint(range_start, range_end) def check_account_type( - user: CustomUserModel, + user: CustomUserModel, ) -> Literal['business_host'] | Literal['business_account'] | Literal['regular'] | Literal['business_admin']: status = AccountStatusSelector(user) if status.is_business_host(): @@ -60,4 +55,3 @@ def count_openrouter_tokens(model_name: str, messages: List[Dict[str, Any]], out input_tokens += len(encoding.encode(message['content'])) output_tokens = len(encoding.encode(output)) return (input_tokens, output_tokens) - @@ -1,164 +0,0 @@ -from typing import List -from uuid import UUID - -from channels.routing import URLRouter -from django.urls import re_path -from ninja import Query, Router - -from authentication.security import SyncAuthBearer -from tools.copywrite.consumers import CopywriteConsumer -from tools.copywrite.exceptions import CopywriteAlreadyGenerated -from tools.copywrite.schemas import ( - CopywriteFilterSchema, - CopywriteQuery, - CreateDraftSchema, - CreateOverridenVariableSchema, - OverridenVariableSchema, - SelfCopywriteSchema, - SelfCopywriteShortSchema, - TemplateCategorySchema, - TemplateCopywriteSchema, - TemplateCopywriteShortSchema, - TemplateFilterSchema, - TemplateSchema, - TemplateShortSchema, - UpdateOverridenVariableSchema, - UpdateSelfCopywriteSchema, -) -from tools.copywrite.services.copywrite import CopywriteService -from tools.copywrite.services.template import TemplateService -from tools.copywrite.tasks import generate as generate_task - -router = Router(tags=['copywrite'], auth=SyncAuthBearer()) - -ws_router = URLRouter( - [ - re_path( - 'copywrites/(?P[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}|)/$', - CopywriteConsumer.as_asgi(), - ) - ] -) - - -@router.get( - 'copywrites/', - tags=['copywrite/copywrites'], - response=List[TemplateCopywriteShortSchema | SelfCopywriteShortSchema], -) -def list_copywrites( - request, - ordering: Query[CopywriteQuery], - filters: CopywriteFilterSchema = Query(...), -): - return CopywriteService.list_copywrites_by_user( - user=request.auth, - filters=filters.get_filter_expression(), - **ordering.model_dump(), - ) - - -@router.post( - 'copywrites/', - tags=['copywrite/copywrites'], - response=TemplateCopywriteSchema | SelfCopywriteSchema, -) -def create_draft(request, data: CreateDraftSchema): - return CopywriteService.create_draft(user=request.auth, **data.model_dump()) - - -@router.get( - 'copywrites/{id}/', - tags=['copywrite/copywrites'], - response=TemplateCopywriteSchema | SelfCopywriteSchema, -) -def get_copywrite_by_id(request, id: UUID): - return CopywriteService.get_copywrite_by_id(copywrite_id=id) - - -@router.put('copywrites/{id}/generate/', tags=['copywrite/copywrites']) -def generate(request, id: UUID): - cp = CopywriteService.get_copywrite_by_id(copywrite_id=id) - if cp.generated: - raise CopywriteAlreadyGenerated - generate_task.delay(copywrite_id=id) - - -@router.put( - 'copywrites/{id}/favourite', - tags=['copywrite/copywrites'], - response={204: None}, -) -def mark_as_favourite(request, id: UUID): - return CopywriteService.mark_as_favourite(copywrite_id=id) - - -@router.delete( - 'copywrites/{id}/favourite', - tags=['copywrite/copywrites'], - response={204: None}, -) -def remove_from_favourite(request, id: UUID): - return CopywriteService.remove_from_favourite(copywrite_id=id) - - -@router.patch('copywrites/{id}/', tags=['copywrite/copywrites'], response={204: None}) -def update_input_content(request, id: UUID, data: UpdateSelfCopywriteSchema): - return CopywriteService.update_input_content(copywrite_id=id, **data.model_dump()) - - -@router.delete('copywrites/{id}/', tags=['copywrite/copywrites'], response={204: None}) -def delete_copywrite(request, id: UUID): - return CopywriteService.delete(copywrite_id=id) - - -@router.post( - 'copywrites/{copywrite_id}/variables/', - tags=['copywrite/variables'], - response=OverridenVariableSchema, -) -def override_variable(request, copywrite_id: UUID, data: CreateOverridenVariableSchema): - return CopywriteService.override_variable(copywrite_id=copywrite_id, **data.model_dump()) - - -@router.put( - 'copywrites/{copywrite_id}/variables/{id}/', - tags=['copywrite/variables'], - response={204: None}, -) -def update_overriden_variable(request, copywrite_id: UUID, id: UUID, data: UpdateOverridenVariableSchema): - return CopywriteService.update_overriden_variable( - copywrite_id=copywrite_id, variable_id=id, **data.model_dump() - ) - - -@router.delete( - 'copywrites/{copywrite_id}/variables/{id}/', - tags=['copywrite/variables'], - response={204: None}, -) -def remove_overriden_variable(request, copywrite_id: UUID, id: UUID): - CopywriteService.remove_overriden_variable(copywrite_id=copywrite_id, variable_id=id) - - -@router.get( - 'templates/categories/', - tags=['copywrite/templates'], - response=List[TemplateCategorySchema], -) -def list_template_categories(request): - return TemplateService.list_categories() - - -@router.get( - 'templates/', - tags=['copywrite/templates'], - response=List[TemplateShortSchema], -) -def list_templates(request, filters: TemplateFilterSchema = Query(...)): - return TemplateService.list_templates(filters=filters.get_filter_expression()) - - -@router.get('templates/{id}/', tags=['copywrite/templates'], response=TemplateSchema) -def get_template_by_id(request, id: UUID): - return TemplateService.get_template_by_id(template_id=id) @@ -1,33 +0,0 @@ -from logging import getLogger - -from channels.generic.websocket import AsyncJsonWebsocketConsumer -from channels_redis.core import RedisChannelLayer -from django.core.cache import cache - -from tools.copywrite.constants import COPYWRITE_WS_KEY - -logger = getLogger(__name__) - - -class CopywriteConsumer(AsyncJsonWebsocketConsumer): - channel_layer: RedisChannelLayer - - async def connect(self): - copywrite_id = f'{self.scope["url_route"]["kwargs"]["copywrite_id"]}' - await self.channel_layer.group_add( - COPYWRITE_WS_KEY % copywrite_id, - self.channel_name, - ) - await self.accept() - if content := cache.get(COPYWRITE_WS_KEY % copywrite_id): - await self.send_json({'chunk': content, 'end': False}) - - async def close(self, code=None, reason=None): - await self.channel_layer.group_discard( - COPYWRITE_WS_KEY % f'{self.scope["url_route"]["kwargs"]["copywrite_id"]}', - self.channel_name, - ) - return await super().close(code, reason) - - async def generate_chunk(self, event_data): - return await self.send_json(event_data) @@ -1,191 +0,0 @@ -from typing import Any, List, Literal, Optional -from uuid import UUID - -from django.db.models import Q -from ninja import Field, FilterSchema, ModelSchema, Schema - -from ml_model.schemas import ModelConfigurationSchema -from tools.copywrite.models import ( - OverridenVariable, - SelfCopywrite, - Template, - TemplateCategory, - TemplateCopywrite, - TemplateVariable, -) - - -class CreateDraftSchema(Schema): - type: str - initial: dict[str, Any] = {} - - -class UpdateSelfCopywriteSchema(ModelSchema): - class Meta: - model = SelfCopywrite - exclude = ( - 'id', - 'favourite', - 'created_at', - 'copywrite_ptr', - 'polymorphic_ctype', - 'output_content', - 'user', - 'deleted', - 'configuration', - ) - fields_optional = '__all__' - - -class UpdateTemplateCopywriteSchema(ModelSchema): - class Meta: - model = SelfCopywrite - exclude = ( - 'id', - 'favourite', - 'created_at', - 'copywrite_ptr', - 'polymorphic_ctype', - 'output_content', - 'user', - ) - fields_optional = '__all__' - - -class TemplateVariableSchema(ModelSchema): - class Meta: - model = TemplateVariable - exclude = ('order', 'sysname', 'template') - - -class TemplateShortSchema(ModelSchema): - class Meta: - model = Template - fields = ('id', 'title', 'description', 'picture') - - -class TemplateCategorySchema(ModelSchema): - class Meta: - model = TemplateCategory - fields = ('title', 'slug') - - -class TemplateSchema(ModelSchema): - variables: List[TemplateVariableSchema] - - class Meta: - model = Template - exclude = ( - 'order', - 'content', - 'category', - 'description', - 'configuration', - 'hidden', - ) - - -class TemplateFilterSchema(FilterSchema): - category: List[str] = Field(None, q='category__slug__in') - - -class BaseCopywriteSchema: - type: str - - -class SelfCopywriteSchema(BaseCopywriteSchema, ModelSchema): - configuration: List[ModelConfigurationSchema] - - class Meta: - model = SelfCopywrite - exclude = ( - 'copywrite_ptr', - 'user', - 'polymorphic_ctype', - 'created_at', - 'deleted', - ) - - -class SelfCopywriteShortSchema(BaseCopywriteSchema, ModelSchema): - draft: bool - label: Optional[str] = None - - class Meta: - model = SelfCopywrite - exclude = ( - 'copywrite_ptr', - 'user', - 'polymorphic_ctype', - 'created_at', - 'output_content', - 'input_content', - ) - - -class OverridenVariableSchema(ModelSchema): - class Meta: - model = OverridenVariable - fields = ('id', 'variable', 'value') - - -class CreateOverridenVariableSchema(ModelSchema): - variable_id: UUID - - class Meta: - model = OverridenVariable - exclude = ('id', 'copywrite', 'variable') - - -class TemplateCopywriteSchema(BaseCopywriteSchema, ModelSchema): - template: TemplateSchema - overriden_variables: List[OverridenVariableSchema] - configuration: List[ModelConfigurationSchema] = [] - - class Meta: - model = TemplateCopywrite - exclude = ( - 'copywrite_ptr', - 'user', - 'polymorphic_ctype', - 'created_at', - 'deleted', - ) - - -class TemplateCopywriteShortSchema(BaseCopywriteSchema, ModelSchema): - template: TemplateShortSchema - draft: bool - label: Optional[str] = None - - class Meta: - model = TemplateCopywrite - exclude = ( - 'copywrite_ptr', - 'user', - 'polymorphic_ctype', - 'created_at', - 'output_content', - ) - - -class CopywriteFilterSchema(FilterSchema): - types: List[Literal['template', 'self']] = [] - - def filter_types(self, value: List[Literal['template', 'self']]): - if value: - return Q(polymorphic_ctype__model__in=list(map(lambda x: f'{x}copywrite', value))) - return Q() - - -class UpdateOverridenVariableSchema(ModelSchema): - class Meta: - model = OverridenVariable - fields = ('value',) - - -class CopywriteQuery(Schema): - sort: List[Literal['favourite', 'created_at', '-favourite', '-created_at']] = [ - '-favourite', - '-created_at', - ] @@ -1,32 +0,0 @@ -from uuid import UUID - -from asgiref.sync import async_to_sync -from celery import shared_task -from channels.layers import get_channel_layer -from channels_redis.core import RedisChannelLayer -from django.core.cache import cache - -from tools.copywrite.constants import COPYWRITE_WS_KEY -from tools.copywrite.services.copywrite import CopywriteService - - -@shared_task -def generate(copywrite_id: UUID): - content = '' - layer: RedisChannelLayer = get_channel_layer() - for chunk in CopywriteService.generate(copywrite_id=copywrite_id): - async_to_sync(layer.group_send)( - COPYWRITE_WS_KEY % copywrite_id, - { - 'type': 'generate.chunk', - 'event_data': {'chunk': chunk, 'end': False}, - }, - ) - content += chunk - cache.set(f'copywrite:{copywrite_id}', content) - async_to_sync(layer.group_send)( - COPYWRITE_WS_KEY % copywrite_id, - {'type': 'generate.chunk', 'event_data': {'chunk': '', 'end': True}}, - ) - CopywriteService.set_output(copywrite_id=copywrite_id, output_content=content) - cache.delete(f'copywrite:{copywrite_id}') @@ -1,5 +1,4 @@ from django.apps import AppConfig -from django.core.signals import setting_changed from django.utils.translation import gettext_lazy as _ @@ -15,17 +14,6 @@ class ChatsConfig(AppConfig): verbose_name = _('Chats') -class CopywriteConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'tools.copywrite' - verbose_name = _('Copywrite') - - def ready(self): - from tools.copywrite import signals as copywrite_signals - - setting_changed.connect(copywrite_signals.copy_model_configuration) - - class PublicAPIConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'tools.public_api' @@ -1,13 +1,22 @@ venv/ +.venv/ + +.gitlab/ .pre-commit-config.yaml .gitignore + data/ +static/ + .*_cache/ __pycache__/ + .idea/ -Makefile .vscode/ -.gitlab-ci.yml -.end.dist +*.egg-info + +Makefile docker-compose.yml docker-compose.debug.yml +.gitlab-ci.yml +.env* @@ -42,15 +42,12 @@ POSTGRES_PASSWORD=postgres POSTGRES_DB=postgres # S3 -MINIO_ACCESS_KEY=testtest -MINIO_SECRET_KEY=testtest MINIO_ENDPOINT=s3:9000 -MINIO_ENDPOINT_USE_HTTPS=false +MINIO_USE_HTTPS=false MINIO_EXTERNAL_ENDPOINT=localhost:9000 MINIO_EXTERNAL_ENDPOINT_USE_HTTPS=false -MINIO_PORT=9000 -MINIO_URL=s3 -MINIO_USE_HTTPS=false +MINIO_ACCESS_KEY=testtest +MINIO_SECRET_KEY=testtest # CELERY CELERY_BROKER_URL=redis://cache-mdb:6379/0 @@ -88,10 +85,6 @@ DJANGO_SUPERUSER_EMAIL=example@root.ru DJANGO_SUPERUSER_USERNAME=root DJANGO_SUPERUSER_PASSWORD=root -# DJANGO CHANNELS -CHANNELS_HOST_MDB=channels-mdb -CHANNELS_PORT_MDB=6379 - LOG_LEVEL=debug DOMAIN=localhost PROVIDER=docker @@ -101,4 +94,11 @@ FEATURE_FLAG_API_URL=https://gitlab.kisulkens.ru/api/v4/feature_flags/unleash/24 FEATURE_FLAG_INSTANCE_ID=glffct-ic8xsVF5eR9BaUySR-_w FEATURE_FLAG_APP_NAME=Production -COMPOSE_FILE=docker-compose.yml:docker-compose.local.yml \ No newline at end of file +# ZROK +ZROK2_API_ENDPOINT=https://zrok2.example.com +ZROK2_ENABLE_TOKEN="your zrok token" + +# DOCKER +COMPOSE_FILE=docker-compose.yml:docker-compose.local.yml +COMPOSE_PROFILES="" # use-tunnel available +BUILDKIT_PROGRESS=plain \ No newline at end of file @@ -2,6 +2,7 @@ __pycache__/ *.pyc .*_cache +*.egg-info # works files .env @@ -23,5 +24,6 @@ data/ celerybeat-schedule .idea/ +.vscode/ scripts/ @@ -1,10 +1,3 @@ -workflow: - rules: - - changes: - - ".gitlab/**/*" - when: never - - when: always - stages: - Build - Deploy @@ -35,6 +28,7 @@ build: ENVIRONMENT: $CI_ENVIRONMENT_TIER DOCKER_TLS_VERIFY: 1 DOCKER_CERT_PATH: "/certs" + BUILDKIT_PROGRESS: quiet COMPOSE_ENV_FILES: $ENV COMPOSE_REMOVE_ORPHANS: true COMPOSE_PROJECT_NAME: $CI_PROJECT_NAME @@ -1,24 +1,19 @@ -FROM python:3.12-slim AS build - -WORKDIR /app - -ARG EXPORT_FLAGS="--only main" - -COPY pyproject.toml poetry.lock /app/ -RUN --mount=type=cache,target=/root/.cache/pip pip install poetry && poetry self add poetry-plugin-export -RUN poetry export ${EXPORT_FLAGS} --output=requirements.txt - FROM python:3.12-slim +# Generic ENV PYTHONFAULTHANDLER=1 \ - PYTHONHASHSEED=random \ - PIP_NO_CACHE_DIR=on \ - PIP_DISABLE_PIP_VERSION_CHECK=on \ - PIP_DEFAULT_TIMEOUT=100 + PYTHONUNBUFFERED=1 -WORKDIR /app +# Build +ENV UV_LINK_MODE=copy \ + UV_COMPILE_BYTECODE=1 \ + UV_PYTHON_DOWNLOADS=0 \ + UV_LOCKED=1 \ + UV_NO_EDITABLE=1 \ + UV_PROJECT_ENVIRONMENT=/app/.venv \ + PATH="/app/.venv/bin:$PATH" -COPY --from=build /app/requirements.txt . +WORKDIR /app RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ --mount=target=/var/cache/apt,type=cache,sharing=locked \ @@ -26,6 +21,14 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ && apt-get update \ && apt-get -y --no-install-recommends install gettext antiword ffmpeg -RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt +RUN --mount=from=ghcr.io/astral-sh/uv,source=/uv,target=/bin/uv \ + --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=uv.lock,target=uv.lock \ + --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ + uv sync --no-install-project COPY . . + +RUN --mount=from=ghcr.io/astral-sh/uv,source=/uv,target=/bin/uv \ + --mount=type=cache,target=/root/.cache/uv \ + uv sync @@ -1,22 +1,39 @@ +x-dev-app-config: &dev-app-config + pull_policy: never + volumes: + - ./:/app + - /app/.venv + services: app: - build: - args: - EXPORT_FLAGS: "--with test --with debug" - volumes: - - ./:/app + <<: *dev-app-config ports: - "8000:8000" - - "5678:5678" + celery: - volumes: - - ./:/app + <<: *dev-app-config + celery-beat: - volumes: - - ./:/app - housekeeper: - volumes: - - ./:/app + <<: *dev-app-config + + init: + <<: *dev-app-config + + create-superuser: + image: ${IMAGE_TAG:-air/backend:latest} + environment: + - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME:?} + - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD:?} + command: + - /bin/sh + - -c + - | + python manage.py createsuperuser --email ${DJANGO_SUPERUSER_EMAIL:?} --no-input --skip-checks || true + depends_on: + init: + condition: service_completed_successfully + <<: *dev-app-config + db: image: postgres:alpine restart: unless-stopped @@ -38,10 +55,68 @@ services: - "9000:9000" - "9001:9001" + tunnel-init: + image: busybox + command: chown -Rc 2171:2171 /mnt/ + user: root + volumes: + - tunnel-data:/mnt + profiles: + - use-tunnel + + tunnel-enable: + image: docker.io/openziti/zrok2 + depends_on: + tunnel-init: + condition: service_completed_successfully + entrypoint: zrok2-enable + environment: + - HOME=/mnt + - ZROK2_ENABLE_TOKEN + - ZROK2_API_ENDPOINT + volumes: + - tunnel-data:/mnt + profiles: + - use-tunnel + + # to enable tunnel after initialization use under container - zrok2 share public http://app:8000 --headless (or another app url) + tunnel: + image: docker.io/openziti/zrok2 + restart: unless-stopped + entrypoint: + - bash + - -c + - rm -f /mnt/.zrok2/agent.socket && exec "$@" + command: + - -- + - zrok2 + - agent + - start + - --console-address + - 0.0.0.0 + - --console-start-port + - "${ZROK2_AGENT_CONSOLE_PORT:-8888}" + - --console-end-port + - "${ZROK2_AGENT_CONSOLE_PORT:-8888}" + depends_on: + tunnel-enable: + condition: service_completed_successfully + ports: + - "${ZROK2_AGENT_CONSOLE_EXTERNAL_PORT:-8888}:${ZROK2_AGENT_CONSOLE_PORT:-8888}" + environment: + - HOME=/mnt + volumes: + - tunnel-data:/mnt + networks: + - default + profiles: + - use-tunnel + volumes: - pgdata: { } - s3data: { } + pgdata: {} + s3data: {} + tunnel-data: {} networks: infrastructure: - external: false \ No newline at end of file + external: false @@ -1,10 +1,25 @@ -x-app-config: &app-config +x-app-image: &app-image image: ${IMAGE_TAG:-air/backend:latest} + +x-default-ulimit: &default-ulimit + ulimits: + nofile: + soft: 98304 + hard: 98304 + +x-app-config: &app-config + <<: [*app-image, *default-ulimit] restart: on-failure:3 + env_file: ${ENV:-.env} environment: - RELEASE - ENVIRONMENT +x-wait-init: &wait-init + depends_on: + init: + condition: service_completed_successfully + x-default-deploy: &default-deploy update_config: parallelism: 1 @@ -22,26 +37,21 @@ x-default-deploy: &default-deploy services: app: - <<: *app-config - ulimits: - nofile: - soft: 98304 - hard: 98304 + <<: [*app-config, *wait-init] build: context: . dockerfile: Dockerfile - volumes: - - static:/code/static command: - /bin/sh - -c - | - python manage.py create_indexes - python manage.py compilemessages + python manage.py compilemessages --locale ru_RU --skip-system-checks python -m uvicorn backend.asgi:application --host 0.0.0.0 --ws wsproto --http httptools --lifespan off --log-level info networks: - default - infrastructure + volumes: + - static:/code/static labels: &app-labels - traefik.enable=true - traefik.${PROVIDER:-docker}.network=${PROXY_NETWORK:-infrastructure} @@ -85,65 +95,55 @@ services: replicas: ${APP_REPLICAS:-1} labels: *app-labels <<: [*default-deploy] - env_file: ${ENV:-.env} - housekeeper: + init: <<: *app-config - ulimits: - nofile: - soft: 98304 - hard: 98304 - deploy: - replicas: 1 - <<: [*default-deploy] command: - /bin/sh - -c - | - python manage.py initialize_buckets - python manage.py collectstatic --no-input + python manage.py create_indexes --skip-system-checks + python manage.py initialize_buckets --skip-checks + python manage.py collectstatic --no-input --skip-checks python manage.py migrate - env_file: ${ENV:-.env} - - celery: - <<: *app-config - ulimits: - nofile: - soft: 98304 - hard: 98304 - command: celery -A backend worker -l INFO --concurrency 3 deploy: replicas: 1 + mode: replicated-job <<: [*default-deploy] - env_file: ${ENV:-.env} + + celery: + <<: [*app-config, *wait-init] environment: - C_FORCE_ROOT=true - RELEASE - ENVIRONMENT + command: celery -A backend worker -l INFO --concurrency 3 + deploy: + replicas: 1 + <<: [*default-deploy] celery-beat: - <<: *app-config - ulimits: - nofile: - soft: 98304 - hard: 98304 + <<: [*app-config, *wait-init] command: celery -A backend beat -l INFO deploy: replicas: 1 <<: [*default-deploy] - env_file: ${ENV:-.env} static-server: image: nginx:alpine - ulimits: - nofile: - soft: 98304 - hard: 98304 command: - /bin/sh - -c - - echo 'server { listen 80 default_server; access_log off; location /${STATIC_PATH_PREFIX:-static} { autoindex on; expires 365d; alias /var/www/static/; } }' > /etc/nginx/conf.d/default.conf - && nginx -g 'daemon off;' + - echo ' + server { + listen 80 default_server; + access_log off; + location /${STATIC_PATH_PREFIX:-/static/} { autoindex on; expires 365d; alias /var/www/static/; } + }' > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;' + networks: + - infrastructure + volumes: + - static:/var/www/static labels: &static-labels - traefik.enable=true - traefik.${PROVIDER:-docker}.network=${PROXY_NETWORK:-infrastructure} @@ -164,38 +164,17 @@ services: replicas: 1 labels: *static-labels <<: [*default-deploy] - networks: - - infrastructure - volumes: - - static:/var/www/static - env_file: ${ENV:-.env} cache-mdb: image: redis:alpine - ulimits: - nofile: - soft: 98304 - hard: 98304 + <<: [*default-ulimit] deploy: replicas: 1 <<: [*default-deploy] celery-mdb: image: redis:alpine - ulimits: - nofile: - soft: 98304 - hard: 98304 - deploy: - replicas: 1 - <<: [*default-deploy] - - channels-mdb: - image: redis:alpine - ulimits: - nofile: - soft: 98304 - hard: 98304 + <<: [*default-ulimit] deploy: replicas: 1 <<: [*default-deploy] @@ -209,4 +188,4 @@ volumes: static: name: "backend-static" locales: - name: "backend-locales" \ No newline at end of file + name: "backend-locales" @@ -1,88 +1,66 @@ -[tool.poetry] -name = "air-backend" -version = "0.1.0" -description = "" -authors = ["guybtw "] -license = "Apache License 2.0" -readme = "README.md" -package-mode = false - -[tool.poetry.dependencies] -python = "^3.12" -django = "5.0.*" -django-cors-headers = "^4.2.0" -djangorestframework-simplejwt = "^5.2.2" -djangorestframework = "^3.14.0" -yookassa = "^3.10.1" -environs = "^9.5.0" -dj-rest-auth = "^4.0.1" -django-celery-beat = "^2.5.0" -minio = "^7.1.15" -drf-spectacular = {extras = ["sidecar"], version = "^0.27.2"} -django-filter = "^23.2" -django-minio-backend = "^3.5.0" -drf-social-oauth2 = "2.1" -social-auth-app-django = "^5.3.0" -django-oauth-toolkit = "^2.3.0" -openpyxl = "^3.1.2" -pypdf2 = "^3.0.1" -mutagen = "^1.47.0" -langserve = {extras = ["client"], version = "^0.0.46"} -django-ordered-model = "^3.7.4" -langchainhub = "^0.1.15" -django-import-export = "^4.0.9" -django-cacheops = "^7.0.2" -celery = {extras = ["redis"], version = "^5.4.0"} -django-redis = "^5.4.0" -django-prometheus = "^2.3.1" -psycopg2-binary = "^2.9.10" -filetype = "^1.2.0" -django-polymorphic = "^3.1.0" -httpx = "^0.28.1" -django-ninja = "^1.3.0" -channels = {extras = ["daphne"], version = "^4.2.0"} -channels-redis = "^4.2.1" -langchain = "^0.3.19" -langchain-openai = "^0.3.6" -langchain-google-genai = "^2.0.9" -tiktoken = "^0.9.0" -langchain-community = "^0.3.17" -docx2txt = "^0.8" -pypandoc = "^1.15" -replicate = "^1.0.4" -ruff = "^0.9.9" -dnspython = "^2.7.0" -deepl = "^1.21.1" -python-docx = "^1.1.2" -pymupdf = "^1.26.1" -pyroscope-io = "^0.8.11" -uvicorn = "^0.35.0" -httptools = "^0.6.4" -wsproto = "^1.2.0" -sentry-sdk = {extras = ["django"], version = "^2.39.0"} -googletrans = "^4.0.2" -pydub = "^0.25.1" -python-dateutil = "^2.9.0.post0" -unleashclient = "^6.4.0" -pillow = "^12.2.0" - - -[tool.poetry.group.test.dependencies] -pytest = "^7.4.0" -factory-boy = "^3.3.0" -snapshottest = "^0.6.0" -coverage = "^7.3.0" -freezegun = "^1.2.2" -pytest-django = "^4.5.2" -pytest-mock = "^3.11.1" -pytest-freezegun = "^0.4.2" -pytest-factoryboy = "^2.5.1" -pytest-cov = "^4.1.0" - +[project] +name = "backend" +dynamic = ["version"] +requires-python = ">=3.12.13" +dependencies = [ + "celery[redis]>=5.6.3", + "deepl>=1.30.0", + "dj-rest-auth>=7.2.0", + "django==5.0.*", + "django-cacheops>=7.2", + "django-celery-beat>=2.9.0", + "django-cors-headers>=4.9.0", + "django-import-export>=4.4.1", + "django-minio-backend>=4.5.0", + "django-ninja>=1.6.2", + "django-ordered-model>=3.7.4", + "django-polymorphic>=4.11.4", + "django-redis>=6.0.0", + "djangorestframework===3.14", + "djangorestframework-simplejwt>=5.5.1", + "dnspython>=2.8.0", + "docx2txt>=0.9", + "drf-social-oauth2>=3.4.1", + "drf-spectacular[sidecar]>=0.29.0", + "environs>=15.0.1", + "filetype>=1.2.0", + "googletrans>=4.0.2", + "httptools>=0.8.0", + "httpx>=0.28.1", + "langchain==0.3.19", + "langchain-google-genai==2.0.9", + "langchain-openai==0.3.6", + "minio>=7.2.20", + "mutagen>=1.47.0", + "openpyxl>=3.1.5", + "pillow>=12.2.0", + "psycopg[binary,pool]>=3.3.4", + "pydub>=0.25.1", + "pymupdf>=1.27.2.3", + "pypandoc>=1.17", + "pypdf2>=3.0.1", + "python-dateutil>=2.9.0.post0", + "python-docx>=1.2.0", + "redis>=6.4.0", + "replicate>=1.0.7", + "sentry-sdk[django]>=2.61.0", + "tiktoken>=0.13.0", + "unleashclient>=6.7.0", + "uvicorn>=0.48.0", + "wsproto>=1.3.2", + "yookassa>=3.10.1", +] -[tool.poetry.group.debug.dependencies] -debugpy = "^1.8.1" +[dependency-groups] +debug = [ + "debugpy>=1.8.20", +] +dev = [ + "ruff>=0.15.15", +] +[tool.setuptools] +py-modules = [] [tool.ruff] exclude = [ @@ -142,22 +120,3 @@ docstring-code-line-length = "dynamic" "__init__.py" = ["E402", "F401"] "**/{tests,docs,tools}/*" = ["E402"] "backend/settings.py" = ["F403", "E402"] - -[tool.pytest.ini_options] -addopts = "-ra" -testpaths = [ - "tests", - "integration", -] - -[tool.coverage.run] -omit = [ - "config/*", - "common/*", - "manage.py", - "**/__init__.py" -] - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api"