diff --git a/docs/admin/machine.rst b/docs/admin/machine.rst index 42c8f7f64f4e..f63a92955c83 100644 --- a/docs/admin/machine.rst +++ b/docs/admin/machine.rst @@ -22,6 +22,8 @@ the project settings: The services translate from the source language as configured at :ref:`component`, see :ref:`component-source_language`. +Per-project automatic suggestion can also be configured via the :ref:`api`. + .. seealso:: :ref:`machine-translation` diff --git a/docs/api.rst b/docs/api.rst index 3bb1980332e3..c8b1b5197209 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1235,6 +1235,30 @@ Projects :>json string full_name: Full name of the contributor :>json string change_count: Number of changes done in the time range + +.. http:get:: /api/projects/{string:project}/machinery_settings/ + + .. versionadded:: 5.9 + + Returns automatic suggestion settings for a project, consisting of the configurations defined for each translation service installed. + + :param project: Project URL slug + :type project: string + :>json object suggestion_settings: Configuration for all installed services. + + +.. http:post:: /api/projects/{string:project}/machinery_settings/ + + .. versionadded:: 5.9 + + Create or update the service configuration for a project. + + :param project: Project URL slug + :type project: string + :form string service: Service name + :form string configuration: Service configuration in JSON + + Components ++++++++++ diff --git a/docs/changes.rst b/docs/changes.rst index a462994e9ceb..6c9c5ea2fa55 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -4,6 +4,10 @@ Weblate 5.9 Not yet released. **New features** +* Per-project :ref:`machine-translation-setup` can now be configured via the Project :ref:`api`. + + * Added :http:get:`/api/projects/{string:project}/machinery_settings/`. + * Added :http:post:`/api/projects/{string:project}/machinery_settings/`. * Translation memory import now supports files with XLIFF, PO and CSV formats, see :ref:`memory-user` and `import_memory` command in :ref:`manage`. * The registration CAPTCHA now includes proof-of-work mechanism ALTCHA. diff --git a/weblate/api/serializers.py b/weblate/api/serializers.py index a737f8a351d6..2961225c4c3c 100644 --- a/weblate/api/serializers.py +++ b/weblate/api/serializers.py @@ -10,6 +10,13 @@ from django.conf import settings from django.db.models import Model +from drf_spectacular.extensions import OpenApiSerializerExtension +from drf_spectacular.plumbing import build_basic_type, build_object_type +from drf_spectacular.utils import ( + OpenApiExample, + extend_schema_serializer, + inline_serializer, +) from rest_framework import serializers from weblate.accounts.models import Subscription @@ -44,6 +51,7 @@ ) if TYPE_CHECKING: + from drf_spectacular.openapi import AutoSchema from rest_framework.request import Request _MT = TypeVar("_MT", bound=Model) # Model Type @@ -397,6 +405,9 @@ class ProjectSerializer(serializers.ModelSerializer[Project]): credits_url = serializers.HyperlinkedIdentityField( view_name="api:project-credits", lookup_field="slug" ) + machinery_settings = serializers.HyperlinkedIdentityField( + view_name="api:project-machinery-settings", lookup_field="slug" + ) class Meta: model = Project @@ -422,6 +433,7 @@ class Meta: "enable_hooks", "language_aliases", "enforced_2fa", + "machinery_settings", ) extra_kwargs = { "url": {"view_name": "api:project-detail", "lookup_field": "slug"} @@ -1513,3 +1525,68 @@ class MetricsSerializer(ReadOnlySerializer): child=serializers.IntegerField(), source="get_celery_queues" ) name = serializers.CharField(source="get_name") + + +@extend_schema_serializer( + examples=[ + OpenApiExample( + "Service settings example", + value={ + "service": "service_name", + "configuration": {"key": "xxxxx", "url": "https://api.service.com/"}, + }, + ) + ] +) +class SingleServiceConfigSerializer(serializers.Serializer): + service = serializers.CharField() + configuration = serializers.DictField() + + +@extend_schema_serializer( + examples=[ + OpenApiExample( + "Service settings example", + value={ + "service1": {"key": "XXXXXXX", "url": "https://api.service.com/"}, + "service2": {"secret": "SECRET_KEY", "credentials": "XXXXXXX"}, + }, + request_only=False, + response_only=True, + ) + ] +) +class ProjectMachinerySettingsSerializer(serializers.Serializer): + def to_representation(self, instance: Project): + return dict(instance.machinery_settings) + + +class ProjectMachinerySettingsSerializerExtension(OpenApiSerializerExtension): + target_class = ProjectMachinerySettingsSerializer + + def map_serializer(self, auto_schema: AutoSchema, direction): + return build_object_type(properties={"service_name": build_basic_type(dict)}) + + +def edit_service_settings_response_serializer( + method: str, *codes +) -> dict[int, serializers.Serializer]: + _serializers = { + 200: inline_serializer( + f"{method}_200_Message_response_serializer", + fields={ + "message": serializers.CharField(), + }, + ), + 201: inline_serializer( + f"{method}_201_Message_response_serializer", + fields={ + "message": serializers.CharField(), + }, + ), + 400: inline_serializer( + f"{method}_400_Error_message_serializer", + fields={"errors": serializers.CharField()}, + ), + } + return {code: _serializers[code] for code in codes} diff --git a/weblate/api/tests.py b/weblate/api/tests.py index 197cd42e8704..418ad7204dd3 100644 --- a/weblate/api/tests.py +++ b/weblate/api/tests.py @@ -7,6 +7,7 @@ from datetime import UTC, datetime, timedelta from io import BytesIO +import responses from django.core.files import File from django.urls import reverse from rest_framework.exceptions import ErrorDetail @@ -2010,6 +2011,247 @@ def test_credits(self) -> None: ) self.assertEqual(response.data, []) + @responses.activate + def test_install_machinery(self) -> None: + """Test the machinery settings API endpoint for various scenarios.""" + from weblate.machinery.tests import AlibabaTranslationTest, DeepLTranslationTest + + # unauthenticated get + self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="get", + code=403, + authenticated=True, + superuser=False, + ) + + # unauthenticated post + self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="post", + code=403, + authenticated=True, + superuser=False, + request={"service": "weblate"}, + ) + + # missing service + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="post", + code=400, + superuser=True, + request={}, + ) + + # unknown service + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="post", + code=400, + superuser=True, + request={"service": "unknown"}, + ) + + # create configuration: no form + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="post", + code=201, + superuser=True, + request={"service": "weblate"}, + ) + + # missing required field + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="post", + code=400, + superuser=True, + request={ + "service": "deepl", + "configuration": {"wrong": ""}, + }, + format="json", + ) + + # invalid field with multipart + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="post", + code=400, + superuser=True, + request={ + "service": "deepl", + "configuration": "{malformed_json", + }, + ) + + # create configuration: valid form with multipart + DeepLTranslationTest.mock_response() + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="post", + code=201, + superuser=True, + request={ + "service": "deepl", + "configuration": '{"key": "x", "url": "https://api.deepl.com/v2/"}', + }, + ) + + # create configuration: valid form with json + AlibabaTranslationTest().mock_response() + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="post", + code=201, + superuser=True, + request={ + "service": "alibaba", + "configuration": { + "key": "alibaba-key-v1", + "secret": "alibaba-secret", + "region": "alibaba-region", + }, + }, + format="json", + ) + + # update configuration: incorrect method + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="post", + code=400, + superuser=True, + request={ + "service": "deepl", + "configuration": { + "key": "deepl-key-v1", + "url": "https://api.deepl.com/v2/", + }, + }, + format="json", + ) + + # update configuration: valid method + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="patch", + code=200, + superuser=True, + request={ + "service": "deepl", + "configuration": { + "key": "deepl-key-v2", + "url": "https://api.deepl.com/v2/", + }, + }, + format="json", + ) + + # list configurations + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="get", + code=200, + superuser=True, + ) + self.assertIn("weblate", response.data) + self.assertEqual(response.data["deepl"]["key"], "deepl-key-v2") + self.assertEqual(response.data["alibaba"]["key"], "alibaba-key-v1") + + # remove configuration + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="patch", + code=200, + superuser=True, + request={"service": "deepl", "configuration": None}, + format="json", + ) + + # check configuration no longer exists + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="get", + code=200, + superuser=True, + ) + self.assertNotIn("deepl", response.data) + + # invalid replace all configurations (missing required config) + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="put", + code=400, + superuser=True, + request={ + "deepl": {"key": "deepl-key-valid", "url": "https://api.deepl.com/v2/"}, + "unknown": {"key": "alibaba-key-invalid"}, + }, + format="json", + ) + + # invalid replace all configurations (missing required config) + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="put", + code=400, + superuser=True, + request={ + "deepl": {"key": "deepl-key-valid", "url": "https://api.deepl.com/v2/"}, + "alibaba": {"key": "alibaba-key-invalid"}, + }, + format="json", + ) + + # replace all configurations + new_config = { + "deepl": {"key": "deepl-key-v3", "url": "https://api.deepl.com/v2/"}, + "alibaba": { + "key": "alibaba-key-v2", + "secret": "alibaba-secret", + "region": "alibaba-region", + }, + } + + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="put", + code=201, + superuser=True, + request=new_config, + format="json", + ) + + # check all configurations + response = self.do_request( + "api:project-machinery-settings", + self.project_kwargs, + method="get", + superuser=True, + ) + + self.assertEqual(new_config, response.data) + class ComponentAPITest(APIBaseTest): def setUp(self) -> None: diff --git a/weblate/api/views.py b/weblate/api/views.py index 865287a35a99..5247c56bb902 100644 --- a/weblate/api/views.py +++ b/weblate/api/views.py @@ -37,6 +37,7 @@ HTTP_200_OK, HTTP_201_CREATED, HTTP_204_NO_CONTENT, + HTTP_400_BAD_REQUEST, HTTP_423_LOCKED, HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -68,23 +69,27 @@ MonolingualUnitSerializer, NewUnitSerializer, NotificationSerializer, + ProjectMachinerySettingsSerializer, ProjectSerializer, RepoRequestSerializer, RoleSerializer, ScreenshotCreateSerializer, ScreenshotFileSerializer, ScreenshotSerializer, + SingleServiceConfigSerializer, StatisticsSerializer, TranslationSerializer, UnitSerializer, UnitWriteSerializer, UploadRequestSerializer, UserStatisticsSerializer, + edit_service_settings_response_serializer, get_reverse_kwargs, ) from weblate.auth.models import AuthenticatedHttpRequest, Group, Role, User from weblate.formats.models import EXPORTERS from weblate.lang.models import Language +from weblate.machinery.models import validate_service_configuration from weblate.memory.models import Memory from weblate.screenshots.models import Screenshot from weblate.trans.exceptions import FileParseError @@ -1010,6 +1015,113 @@ def file(self, request: Request, **kwargs): name=instance.slug, ) + @extend_schema( + responses=ProjectMachinerySettingsSerializer, + methods=["GET"], + description="List machinery settings for a project.", + ) + @extend_schema( + request=SingleServiceConfigSerializer, + responses=edit_service_settings_response_serializer("post", 201, 400), + methods=["POST"], + description="Install a new machinery service", + ) + @extend_schema( + request=SingleServiceConfigSerializer, + responses=edit_service_settings_response_serializer("patch", 200, 400), + methods=["PATCH"], + description="Partially update a single service. Leave configuration blank to remove the service", + ) + @extend_schema( + request=ProjectMachinerySettingsSerializer, + responses=edit_service_settings_response_serializer("put", 200, 400), + methods=["PUT"], + description="Replace configuration for all services.", + ) + @action(detail=True, methods=["get", "post", "patch", "put"]) + def machinery_settings(self, request: Request, **kwargs): + """List or create/update machinery configuration for a project.""" + project = self.get_object() + + if not request.user.has_perm("project.edit", project): + self.permission_denied( + request, "Can not retrieve/edit machinery configuration" + ) + + if request.method in {"POST", "PATCH"}: + try: + service_name = request.data["service"] + except KeyError: + return Response( + {"errors": ["Missing service name"]}, status=HTTP_400_BAD_REQUEST + ) + + service, configuration, errors = validate_service_configuration( + service_name, request.data.get("configuration", "{}") + ) + + if service is None or errors: + return Response({"errors": errors}, status=HTTP_400_BAD_REQUEST) + + if request.method == "PATCH": + if configuration: + # update a configuration + project.machinery_settings[service_name] = configuration + project.save(update_fields=["machinery_settings"]) + return Response( + {"message": f"Service updated: {service.name}"}, + status=HTTP_200_OK, + ) + # remove a configuration + project.machinery_settings.pop(service_name, None) + project.save(update_fields=["machinery_settings"]) + return Response( + {"message": f"Service removed: {service.name}"}, + status=HTTP_200_OK, + ) + + if request.method == "POST": + if service_name in project.machinery_settings: + return Response( + {"errors": ["Service already exists"]}, + status=HTTP_400_BAD_REQUEST, + ) + + project.machinery_settings[service_name] = configuration + project.save(update_fields=["machinery_settings"]) + return Response( + {"message": f"Service installed: {service.name}"}, + status=HTTP_201_CREATED, + ) + + elif request.method == "PUT": + # replace all service configuration + valid_configurations: dict[str, dict] = {} + for service_name, configuration in request.data.items(): + service, configuration, errors = validate_service_configuration( + service_name, configuration + ) + + if service is None or errors: + return Response({"errors": errors}, status=HTTP_400_BAD_REQUEST) + + valid_configurations[service_name] = configuration + + project.machinery_settings = valid_configurations + project.save(update_fields=["machinery_settings"]) + return Response( + { + "message": f"Services installed: {', '.join(valid_configurations.keys())}" + }, + status=HTTP_201_CREATED, + ) + + # GET method + return Response( + data=ProjectMachinerySettingsSerializer(project).data, + status=HTTP_200_OK, + ) + class ComponentViewSet( MultipleFieldViewSet, UpdateModelMixin, DestroyModelMixin, CreditsMixin diff --git a/weblate/machinery/management/commands/install_machinery.py b/weblate/machinery/management/commands/install_machinery.py index cb0594d23590..7611eec5a2c3 100644 --- a/weblate/machinery/management/commands/install_machinery.py +++ b/weblate/machinery/management/commands/install_machinery.py @@ -2,12 +2,10 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -import json - from django.core.management.base import CommandError from weblate.configuration.models import Setting, SettingCategory -from weblate.machinery.models import MACHINERY +from weblate.machinery.models import validate_service_configuration from weblate.utils.management.base import BaseCommand @@ -26,31 +24,17 @@ def add_arguments(self, parser) -> None: help="Update existing service configuration", ) - def validate_form(self, form) -> None: - if not form.is_valid(): - for error in form.non_field_errors(): + def handle(self, *args, **options) -> None: + service, configuration, errors = validate_service_configuration( + options["service"], options["configuration"] + ) + + if service is None or errors: + for error in errors: self.stderr.write(error) - for field in form: - for error in field.errors: - self.stderr.write(f"Error in {field.name}: {error}") msg = "Invalid add-on configuration!" raise CommandError(msg) - def handle(self, *args, **options) -> None: - try: - service = MACHINERY[options["service"]] - except KeyError as error: - msg = "Service not found: {}".format(options["service"]) - raise CommandError(msg) from error - try: - configuration = json.loads(options["configuration"]) - except ValueError as error: - msg = f"Invalid service configuration: {error}" - raise CommandError(msg) from error - if service.settings_form is not None: - form = service.settings_form(service, data=configuration) - self.validate_form(form) - setting, created = Setting.objects.get_or_create( category=SettingCategory.MT, name=options["service"], diff --git a/weblate/machinery/models.py b/weblate/machinery/models.py index ede738232703..c9cc74791ab0 100644 --- a/weblate/machinery/models.py +++ b/weblate/machinery/models.py @@ -2,6 +2,8 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +import json + from appconf import AppConf from weblate.utils.classloader import ClassLoader @@ -50,3 +52,47 @@ class WeblateConf(AppConf): class Meta: prefix = "" + + +def validate_service_configuration( + service_name: str, configuration: str | dict +) -> tuple[BatchMachineTranslation | None, dict, list[str]]: + """ + Validate given service configuration. + + :param service_name: Name of the service as defined in WEBLATE_MACHINERY + :param configuration: JSON encoded configuration for the service + :return: A tuple containing the validated service class, configuration + and a list of errors + :raises ValueError: When service is not found or configuration is invalid + """ + try: + service = MACHINERY[service_name] + except KeyError: + msg = f"Service not found: {service_name}" + return None, {}, [msg] + + if isinstance(configuration, str): + try: + service_configuration = json.loads(configuration) + except ValueError as error: + msg = f"Invalid service configuration ({service_name}): {error}" + return service, {}, [msg] + else: + service_configuration = configuration + + errors = [] + if service.settings_form is not None: + form = service.settings_form(service, data=service_configuration) + # validate form + if not form.is_valid(): + errors.extend(list(form.non_field_errors())) + + for field in form: + errors.extend( + [ + f"Error in {field.name} ({service_name}): {error}" + for error in field.errors + ] + ) + return service, service_configuration, errors diff --git a/weblate/machinery/tests.py b/weblate/machinery/tests.py index ccda2092d7c4..8fe11356a487 100644 --- a/weblate/machinery/tests.py +++ b/weblate/machinery/tests.py @@ -2628,7 +2628,7 @@ def test_list_addons(self) -> None: call_command("list_machinery", stdout=output) self.assertIn("DeepL", output.getvalue()) - def test_install_no_form(self) -> None: + def test_valid_install_no_form(self) -> None: output = StringIO() call_command( "install_machinery", @@ -2639,6 +2639,17 @@ def test_install_no_form(self) -> None: ) self.assertIn("Service installed: Weblate", output.getvalue()) + def test_install_unknown_service(self) -> None: + output = StringIO() + with self.assertRaises(CommandError): + call_command( + "install_machinery", + "--service", + "unknown", + stdout=output, + stderr=output, + ) + def test_install_missing_form(self) -> None: output = StringIO() with self.assertRaises(CommandError): @@ -2672,7 +2683,27 @@ def test_install_valid_form(self) -> None: "--service", "deepl", "--configuration", - '{"key": "x", "url": "https://api.deepl.com/v2/"}', + '{"key": "x1", "url": "https://api.deepl.com/v2/"}', stdout=output, stderr=output, ) + self.assertTrue( + Setting.objects.filter(category=SettingCategory.MT, name="deepl").exists() + ) + + # update configuration + call_command( + "install_machinery", + "--service", + "deepl", + "--configuration", + '{"key": "x2", "url": "https://api.deepl.com/v2/"}', + "--update", + stdout=output, + stderr=output, + ) + + setting = Setting.objects.get(category=SettingCategory.MT, name="deepl") + self.assertEqual( + setting.value, {"key": "x2", "url": "https://api.deepl.com/v2/"} + )