diff --git a/docs/api.rst b/docs/api.rst index 65aeb4ea69ac..51458690e18d 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1079,6 +1079,7 @@ Projects :type project: string :form file zipfile: ZIP file to upload into Weblate for translations initialization :form file docfile: Document to translate + :form string from_component: Optional source component reference used to duplicate the new component. Accepts either a numeric component ID or a full Weblate component path. When provided, the new component inherits the source component configuration and translations; values explicitly supplied in the request override the inherited configuration. If ``repo`` is omitted, a local repository is created to avoid writing seeded content back to the source repository by default. :form boolean disable_autoshare: Disables automatic repository sharing via :ref:`internal-urls`. :json object result: Created component object; see :http:get:`/api/components/(string:project)/(string:component)/` @@ -1163,11 +1164,11 @@ Projects Content-Length: 20 { + "from_component": "hello/weblate", "file_format": "po", "filemask": "po/*.po", "name": "Weblate", "slug": "weblate", - "repo": "weblate://weblate/hello", "template": "", "new_base": "po/hello.pot", "vcs": "git" @@ -1805,6 +1806,7 @@ Components :param component: Component URL slug :type component: string :json object result: new translation object created **CURL example:** @@ -1827,7 +1829,10 @@ Components Authorization: Token TOKEN Content-Length: 20 - {"language_code": "cs"} + { + "language_code": "cs", + "from_component": ["hello/weblate", 123] + } **JSON response example:** diff --git a/docs/changes.rst b/docs/changes.rst index 6487a588efa3..b19e7443a970 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -9,6 +9,7 @@ Weblate 5.17 * Shared components can now be categorized within the target project. * :ref:`api` supports specifying a category when sharing a component via ``category_id`` parameter. * Added :ref:`addon-weblate.gettext.xgettext`, :ref:`addon-weblate.gettext.django`, and :ref:`addon-weblate.gettext.sphinx` to update POT files with configurable update cadence. +* Added ``from_component`` support to the REST API for creating components from existing component content and for creating translations seeded by automatic translation from existing components. .. rubric:: Improvements diff --git a/docs/specs/openapi.yaml b/docs/specs/openapi.yaml index 8fde8635a795..f040931ec354 100644 --- a/docs/specs/openapi.yaml +++ b/docs/specs/openapi.yaml @@ -46796,6 +46796,7 @@ components: - $ref: '#/components/schemas/ApiComponentsAddonsCreateVariantRegexErrorComponent' - $ref: '#/components/schemas/ApiComponentsAddonsCreateZipfileErrorComponent' - $ref: '#/components/schemas/ApiComponentsAddonsCreateDocfileErrorComponent' + - $ref: '#/components/schemas/ApiComponentsAddonsCreateFromComponentErrorComponent' - $ref: '#/components/schemas/ApiComponentsAddonsCreateIsGlossaryErrorComponent' - $ref: '#/components/schemas/ApiComponentsAddonsCreateGlossaryColorErrorComponent' - $ref: '#/components/schemas/ApiComponentsAddonsCreateDisableAutoshareErrorComponent' @@ -46860,6 +46861,7 @@ components: variant_regex: '#/components/schemas/ApiComponentsAddonsCreateVariantRegexErrorComponent' zipfile: '#/components/schemas/ApiComponentsAddonsCreateZipfileErrorComponent' docfile: '#/components/schemas/ApiComponentsAddonsCreateDocfileErrorComponent' + from_component: '#/components/schemas/ApiComponentsAddonsCreateFromComponentErrorComponent' is_glossary: '#/components/schemas/ApiComponentsAddonsCreateIsGlossaryErrorComponent' glossary_color: '#/components/schemas/ApiComponentsAddonsCreateGlossaryColorErrorComponent' disable_autoshare: '#/components/schemas/ApiComponentsAddonsCreateDisableAutoshareErrorComponent' @@ -46951,6 +46953,34 @@ components: - attr - code - detail + ApiComponentsAddonsCreateFromComponentErrorComponent: + type: object + properties: + attr: + enum: + - from_component + type: string + description: '* `from_component` - from_component' + code: + enum: + - blank + - invalid + - 'null' + - null_characters_not_allowed + - surrogate_characters_not_allowed + type: string + description: |- + * `blank` - blank + * `invalid` - invalid + * `null` - null + * `null_characters_not_allowed` - null_characters_not_allowed + * `surrogate_characters_not_allowed` - surrogate_characters_not_allowed + detail: + type: string + required: + - attr + - code + - detail ApiComponentsAddonsCreateGitExportErrorComponent: type: object properties: @@ -48548,6 +48578,7 @@ components: - $ref: '#/components/schemas/ApiComponentsLinksCreateVariantRegexErrorComponent' - $ref: '#/components/schemas/ApiComponentsLinksCreateZipfileErrorComponent' - $ref: '#/components/schemas/ApiComponentsLinksCreateDocfileErrorComponent' + - $ref: '#/components/schemas/ApiComponentsLinksCreateFromComponentErrorComponent' - $ref: '#/components/schemas/ApiComponentsLinksCreateIsGlossaryErrorComponent' - $ref: '#/components/schemas/ApiComponentsLinksCreateGlossaryColorErrorComponent' - $ref: '#/components/schemas/ApiComponentsLinksCreateDisableAutoshareErrorComponent' @@ -48612,6 +48643,7 @@ components: variant_regex: '#/components/schemas/ApiComponentsLinksCreateVariantRegexErrorComponent' zipfile: '#/components/schemas/ApiComponentsLinksCreateZipfileErrorComponent' docfile: '#/components/schemas/ApiComponentsLinksCreateDocfileErrorComponent' + from_component: '#/components/schemas/ApiComponentsLinksCreateFromComponentErrorComponent' is_glossary: '#/components/schemas/ApiComponentsLinksCreateIsGlossaryErrorComponent' glossary_color: '#/components/schemas/ApiComponentsLinksCreateGlossaryColorErrorComponent' disable_autoshare: '#/components/schemas/ApiComponentsLinksCreateDisableAutoshareErrorComponent' @@ -48703,6 +48735,34 @@ components: - attr - code - detail + ApiComponentsLinksCreateFromComponentErrorComponent: + type: object + properties: + attr: + enum: + - from_component + type: string + description: '* `from_component` - from_component' + code: + enum: + - blank + - invalid + - 'null' + - null_characters_not_allowed + - surrogate_characters_not_allowed + type: string + description: |- + * `blank` - blank + * `invalid` - invalid + * `null` - null + * `null_characters_not_allowed` - null_characters_not_allowed + * `surrogate_characters_not_allowed` - surrogate_characters_not_allowed + detail: + type: string + required: + - attr + - code + - detail ApiComponentsLinksCreateGitExportErrorComponent: type: object properties: @@ -50390,6 +50450,7 @@ components: - $ref: '#/components/schemas/ApiComponentsPartialUpdateVariantRegexErrorComponent' - $ref: '#/components/schemas/ApiComponentsPartialUpdateZipfileErrorComponent' - $ref: '#/components/schemas/ApiComponentsPartialUpdateDocfileErrorComponent' + - $ref: '#/components/schemas/ApiComponentsPartialUpdateFromComponentErrorComponent' - $ref: '#/components/schemas/ApiComponentsPartialUpdateIsGlossaryErrorComponent' - $ref: '#/components/schemas/ApiComponentsPartialUpdateGlossaryColorErrorComponent' - $ref: '#/components/schemas/ApiComponentsPartialUpdateDisableAutoshareErrorComponent' @@ -50454,6 +50515,7 @@ components: variant_regex: '#/components/schemas/ApiComponentsPartialUpdateVariantRegexErrorComponent' zipfile: '#/components/schemas/ApiComponentsPartialUpdateZipfileErrorComponent' docfile: '#/components/schemas/ApiComponentsPartialUpdateDocfileErrorComponent' + from_component: '#/components/schemas/ApiComponentsPartialUpdateFromComponentErrorComponent' is_glossary: '#/components/schemas/ApiComponentsPartialUpdateIsGlossaryErrorComponent' glossary_color: '#/components/schemas/ApiComponentsPartialUpdateGlossaryColorErrorComponent' disable_autoshare: '#/components/schemas/ApiComponentsPartialUpdateDisableAutoshareErrorComponent' @@ -50545,6 +50607,34 @@ components: - attr - code - detail + ApiComponentsPartialUpdateFromComponentErrorComponent: + type: object + properties: + attr: + enum: + - from_component + type: string + description: '* `from_component` - from_component' + code: + enum: + - blank + - invalid + - 'null' + - null_characters_not_allowed + - surrogate_characters_not_allowed + type: string + description: |- + * `blank` - blank + * `invalid` - invalid + * `null` - null + * `null_characters_not_allowed` - null_characters_not_allowed + * `surrogate_characters_not_allowed` - surrogate_characters_not_allowed + detail: + type: string + required: + - attr + - code + - detail ApiComponentsPartialUpdateGitExportErrorComponent: type: object properties: @@ -52218,6 +52308,7 @@ components: - $ref: '#/components/schemas/ApiComponentsTranslationsCreateVariantRegexErrorComponent' - $ref: '#/components/schemas/ApiComponentsTranslationsCreateZipfileErrorComponent' - $ref: '#/components/schemas/ApiComponentsTranslationsCreateDocfileErrorComponent' + - $ref: '#/components/schemas/ApiComponentsTranslationsCreateFromComponentErrorComponent' - $ref: '#/components/schemas/ApiComponentsTranslationsCreateIsGlossaryErrorComponent' - $ref: '#/components/schemas/ApiComponentsTranslationsCreateGlossaryColorErrorComponent' - $ref: '#/components/schemas/ApiComponentsTranslationsCreateDisableAutoshareErrorComponent' @@ -52282,6 +52373,7 @@ components: variant_regex: '#/components/schemas/ApiComponentsTranslationsCreateVariantRegexErrorComponent' zipfile: '#/components/schemas/ApiComponentsTranslationsCreateZipfileErrorComponent' docfile: '#/components/schemas/ApiComponentsTranslationsCreateDocfileErrorComponent' + from_component: '#/components/schemas/ApiComponentsTranslationsCreateFromComponentErrorComponent' is_glossary: '#/components/schemas/ApiComponentsTranslationsCreateIsGlossaryErrorComponent' glossary_color: '#/components/schemas/ApiComponentsTranslationsCreateGlossaryColorErrorComponent' disable_autoshare: '#/components/schemas/ApiComponentsTranslationsCreateDisableAutoshareErrorComponent' @@ -52373,6 +52465,34 @@ components: - attr - code - detail + ApiComponentsTranslationsCreateFromComponentErrorComponent: + type: object + properties: + attr: + enum: + - from_component + type: string + description: '* `from_component` - from_component' + code: + enum: + - blank + - invalid + - 'null' + - null_characters_not_allowed + - surrogate_characters_not_allowed + type: string + description: |- + * `blank` - blank + * `invalid` - invalid + * `null` - null + * `null_characters_not_allowed` - null_characters_not_allowed + * `surrogate_characters_not_allowed` - surrogate_characters_not_allowed + detail: + type: string + required: + - attr + - code + - detail ApiComponentsTranslationsCreateGitExportErrorComponent: type: object properties: @@ -53949,6 +54069,7 @@ components: - $ref: '#/components/schemas/ApiComponentsUpdateVariantRegexErrorComponent' - $ref: '#/components/schemas/ApiComponentsUpdateZipfileErrorComponent' - $ref: '#/components/schemas/ApiComponentsUpdateDocfileErrorComponent' + - $ref: '#/components/schemas/ApiComponentsUpdateFromComponentErrorComponent' - $ref: '#/components/schemas/ApiComponentsUpdateIsGlossaryErrorComponent' - $ref: '#/components/schemas/ApiComponentsUpdateGlossaryColorErrorComponent' - $ref: '#/components/schemas/ApiComponentsUpdateDisableAutoshareErrorComponent' @@ -54013,6 +54134,7 @@ components: variant_regex: '#/components/schemas/ApiComponentsUpdateVariantRegexErrorComponent' zipfile: '#/components/schemas/ApiComponentsUpdateZipfileErrorComponent' docfile: '#/components/schemas/ApiComponentsUpdateDocfileErrorComponent' + from_component: '#/components/schemas/ApiComponentsUpdateFromComponentErrorComponent' is_glossary: '#/components/schemas/ApiComponentsUpdateIsGlossaryErrorComponent' glossary_color: '#/components/schemas/ApiComponentsUpdateGlossaryColorErrorComponent' disable_autoshare: '#/components/schemas/ApiComponentsUpdateDisableAutoshareErrorComponent' @@ -54104,6 +54226,34 @@ components: - attr - code - detail + ApiComponentsUpdateFromComponentErrorComponent: + type: object + properties: + attr: + enum: + - from_component + type: string + description: '* `from_component` - from_component' + code: + enum: + - blank + - invalid + - 'null' + - null_characters_not_allowed + - surrogate_characters_not_allowed + type: string + description: |- + * `blank` - blank + * `invalid` - invalid + * `null` - null + * `null_characters_not_allowed` - null_characters_not_allowed + * `surrogate_characters_not_allowed` - surrogate_characters_not_allowed + detail: + type: string + required: + - attr + - code + - detail ApiComponentsUpdateGitExportErrorComponent: type: object properties: @@ -58810,6 +58960,7 @@ components: - $ref: '#/components/schemas/ApiProjectsComponentsCreateVariantRegexErrorComponent' - $ref: '#/components/schemas/ApiProjectsComponentsCreateZipfileErrorComponent' - $ref: '#/components/schemas/ApiProjectsComponentsCreateDocfileErrorComponent' + - $ref: '#/components/schemas/ApiProjectsComponentsCreateFromComponentErrorComponent' - $ref: '#/components/schemas/ApiProjectsComponentsCreateIsGlossaryErrorComponent' - $ref: '#/components/schemas/ApiProjectsComponentsCreateGlossaryColorErrorComponent' - $ref: '#/components/schemas/ApiProjectsComponentsCreateDisableAutoshareErrorComponent' @@ -58874,6 +59025,7 @@ components: variant_regex: '#/components/schemas/ApiProjectsComponentsCreateVariantRegexErrorComponent' zipfile: '#/components/schemas/ApiProjectsComponentsCreateZipfileErrorComponent' docfile: '#/components/schemas/ApiProjectsComponentsCreateDocfileErrorComponent' + from_component: '#/components/schemas/ApiProjectsComponentsCreateFromComponentErrorComponent' is_glossary: '#/components/schemas/ApiProjectsComponentsCreateIsGlossaryErrorComponent' glossary_color: '#/components/schemas/ApiProjectsComponentsCreateGlossaryColorErrorComponent' disable_autoshare: '#/components/schemas/ApiProjectsComponentsCreateDisableAutoshareErrorComponent' @@ -58965,6 +59117,34 @@ components: - attr - code - detail + ApiProjectsComponentsCreateFromComponentErrorComponent: + type: object + properties: + attr: + enum: + - from_component + type: string + description: '* `from_component` - from_component' + code: + enum: + - blank + - invalid + - 'null' + - null_characters_not_allowed + - surrogate_characters_not_allowed + type: string + description: |- + * `blank` - blank + * `invalid` - invalid + * `null` - null + * `null_characters_not_allowed` - null_characters_not_allowed + * `surrogate_characters_not_allowed` - surrogate_characters_not_allowed + detail: + type: string + required: + - attr + - code + - detail ApiProjectsComponentsCreateGitExportErrorComponent: type: object properties: @@ -66720,6 +66900,9 @@ components: docfile: type: string format: uri + from_component: + type: string + writeOnly: true addons: type: array items: @@ -70031,6 +70214,9 @@ components: docfile: type: string format: uri + from_component: + type: string + writeOnly: true addons: type: array items: diff --git a/weblate/api/serializers.py b/weblate/api/serializers.py index 1c408946512d..26cbfeedcfbf 100644 --- a/weblate/api/serializers.py +++ b/weblate/api/serializers.py @@ -4,8 +4,9 @@ from __future__ import annotations -from copy import copy -from typing import TYPE_CHECKING, TypeVar, cast +from copy import copy, deepcopy +from dataclasses import dataclass +from typing import TYPE_CHECKING, ClassVar, TypeVar, cast from zipfile import BadZipfile from django.conf import settings @@ -32,7 +33,9 @@ from weblate.lang.models import Language, Plural from weblate.memory.models import Memory from weblate.screenshots.models import Screenshot +from weblate.trans.component_seed import validate_component_snapshot_paths from weblate.trans.defines import BRANCH_LENGTH, LANGUAGE_NAME_LENGTH, REPO_LENGTH +from weblate.trans.file_format_params import strip_unused_file_format_params from weblate.trans.models import ( AutoComponentList, Category, @@ -46,7 +49,11 @@ Unit, ) from weblate.trans.models.translation import NewUnitParams -from weblate.trans.util import check_upload_method_permissions, cleanup_repo_url +from weblate.trans.util import ( + check_upload_method_permissions, + cleanup_repo_url, + is_repo_link, +) from weblate.utils.site import get_site_url from weblate.utils.state import STATE_READONLY, StringState from weblate.utils.validators import validate_bitmap @@ -66,6 +73,56 @@ _MT = TypeVar("_MT", bound=Model) # Model Type +@dataclass +class ComponentReference: + value: str + + +def resolve_component_reference( + queryset, value: str | int | ComponentReference +) -> Component: + """Resolve component reference by numeric ID or full Weblate path.""" + if isinstance(value, Component): + return value + if isinstance(value, ComponentReference): + value = value.value + if isinstance(value, int): + return queryset.get(pk=value) + + text = str(value).strip() + if text.isdigit(): + return queryset.get(pk=int(text)) + return queryset.get_by_path(text) + + +def resolve_component_reference_with_error( + queryset, value, field_name: str +) -> Component: + try: + return resolve_component_reference(queryset, value) + except Component.DoesNotExist as error: + raise serializers.ValidationError( + {field_name: "Component not found."} + ) from error + + +class ComponentReferenceField(serializers.CharField): + def to_internal_value(self, data): + text = super().to_internal_value(str(data)) + return ComponentReference(text) + + +class ComponentReferenceListField(serializers.ListField): + child = ComponentReferenceField() + + def get_value(self, dictionary): + if hasattr(dictionary, "getlist"): + values = dictionary.getlist(self.field_name) + if values: + return values + return super().get_value(dictionary) + + def get_reverse_kwargs( obj, lookup_field: tuple[str, ...], strip_parts: int = 0 ) -> dict[str, str] | None: @@ -714,6 +771,71 @@ def get_url(self, obj, view_name, request: Request, format): # noqa: A002 class ComponentSerializer(RemovableSerializer[Component]): + forbidden_from_component_override_fields: ClassVar[frozenset[str]] = frozenset( + { + "source_language", + "filemask", + "template", + "edit_template", + "intermediate", + "new_base", + "file_format", + "file_format_params", + "language_code_style", + "language_regex", + "key_filter", + "variant_regex", + "manage_units", + } + ) + duplicated_component_fields = ( + "source_language", + "vcs", + "repo", + "branch", + "push", + "push_branch", + "filemask", + "screenshot_filemask", + "template", + "edit_template", + "intermediate", + "new_base", + "file_format", + "license", + "agreement", + "new_lang", + "language_code_style", + "check_flags", + "priority", + "enforced_checks", + "restricted", + "repoweb", + "report_source_bugs", + "merge_style", + "commit_message", + "add_message", + "delete_message", + "merge_message", + "addon_message", + "pull_message", + "allow_translation_propagation", + "manage_units", + "enable_suggestions", + "suggestion_voting", + "suggestion_autoaccept", + "push_on_commit", + "commit_pending_age", + "auto_lock_error", + "language_regex", + "key_filter", + "secondary_language", + "variant_regex", + "is_glossary", + "glossary_color", + "hide_glossary_matches", + "contribute_project_tm", + ) web_url = AbsoluteURLField(source="get_absolute_url", read_only=True) project = ProjectSerializer(read_only=True) repository_url = MultiFieldHyperlinkedIdentityField( @@ -753,6 +875,7 @@ class ComponentSerializer(RemovableSerializer[Component]): zipfile = serializers.FileField(required=False) docfile = serializers.FileField(required=False) + from_component = ComponentReferenceField(required=False, write_only=True) disable_autoshare = serializers.BooleanField(required=False) enforced_checks = serializers.JSONField(required=False) @@ -843,6 +966,7 @@ class Meta: "variant_regex", "zipfile", "docfile", + "from_component", "addons", "is_glossary", "glossary_color", @@ -897,6 +1021,15 @@ def to_internal_value(self, data): # Preprocess to inject params based on content data = data.copy() + source_component = None + if "from_component" in data and "docfile" not in data and "zipfile" not in data: + source_component = resolve_component_reference_with_error( + Component.objects.filter_access(self.context["request"].user), + data["from_component"], + "from_component", + ) + self.populate_from_component_input_defaults(data, source_component) + # Provide a reasonable default if "manage_units" not in data and data.get("template"): data["manage_units"] = "1" @@ -932,21 +1065,175 @@ def to_internal_value(self, data): if "category" not in result: result["category"] = None + if source_component is not None: + result["from_component"] = source_component + return result + def populate_from_component_input_defaults(self, data, source_component: Component): + defaults = { + "filemask": source_component.filemask, + "file_format": source_component.file_format, + } + if "repo" in data: + defaults["vcs"] = source_component.vcs + else: + defaults["repo"] = ( + source_component.repo or source_component.get_repo_link_url() + ) + defaults["vcs"] = source_component.vcs + for field, value in defaults.items(): + if field not in data: + data[field] = value + + def apply_from_component_defaults(self, attrs, source_component: Component): + project = attrs.get("project") or self.context.get("project") + + for field in self.duplicated_component_fields: + if field in attrs: + continue + if "repo" in self.initial_data and field in { + "branch", + "push", + "push_branch", + }: + continue + value = getattr(source_component, field) + if isinstance(value, list | dict): + value = deepcopy(value) + attrs[field] = value + + if ( + attrs.get("category") is None + and "category" not in self.initial_data + and project is not None + ): + attrs["category"] = ( + source_component.category + if source_component.project_id == project.pk + else None + ) + + if "file_format_params" not in attrs and source_component.file_format_params: + attrs["file_format_params"] = strip_unused_file_format_params( + attrs.get("file_format", source_component.file_format), + deepcopy(source_component.file_format_params), + ) + + return attrs + + def validate_from_component_overrides(self, attrs, source_component: Component): + forbidden_fields = sorted( + self.forbidden_from_component_override_fields.intersection( + self.initial_data + ) + ) + if forbidden_fields: + raise serializers.ValidationError( + dict.fromkeys( + forbidden_fields, + "This field can not be overridden when using from_component.", + ) + ) + + if "repo" not in self.initial_data: + incompatible_fields = sorted( + {"vcs", "branch", "push", "push_branch"}.intersection(self.initial_data) + ) + if incompatible_fields: + raise serializers.ValidationError( + dict.fromkeys( + incompatible_fields, + "This field requires repo when using from_component.", + ) + ) + return + + source_repo = source_component.repo or source_component.get_repo_link_url() + if attrs.get("repo") != source_repo: + return + + target_branch = self.get_effective_component_branch( + attrs.get("vcs", source_component.vcs), + attrs["repo"], + attrs.get("branch", ""), + ) + source_branch = self.get_effective_component_branch( + source_component.vcs, + source_repo, + source_component.branch, + ) + if target_branch == source_branch: + raise serializers.ValidationError( + {"branch": "Cloning into the same repository branch is not allowed."} + ) + + @staticmethod + def get_effective_component_branch(vcs: str, repo: str, branch: str) -> str: + if branch or is_repo_link(repo): + return branch + component = Component(vcs=vcs, repo=repo) + return component.repository_class.get_remote_branch(repo) + + @staticmethod + def validate_local_from_component_instance(instance: Component) -> None: + instance.clean_unique_together() + instance.clean_category() + instance.clean_new_lang() + instance.clean_file_format_params() + def validate(self, attrs): # Handle non-component args disable_autoshare = attrs.pop("disable_autoshare", False) docfile = attrs.pop("docfile", None) zipfile = attrs.pop("zipfile", None) + from_component = attrs.pop("from_component", None) # Restrict create fields on patching - if self.instance and (docfile is not None or zipfile is not None): - field = "docfile" if docfile is not None else "zipfile" + if self.instance and ( + docfile is not None or zipfile is not None or from_component is not None + ): + field = ( + "docfile" + if docfile is not None + else "zipfile" + if zipfile is not None + else "from_component" + ) raise serializers.ValidationError( {field: "This field is for creation only, use /file/ instead."} ) + source_component = None + if from_component is not None: + source_component = resolve_component_reference_with_error( + Component.objects.filter_access(self.context["request"].user), + from_component, + "from_component", + ) + if not self.context["request"].user.has_perm( + "component.edit", source_component + ): + raise serializers.ValidationError( + { + "from_component": "You do not have permission to use this component." + } + ) + if docfile is not None or zipfile is not None: + raise serializers.ValidationError( + { + "from_component": "This field can not be combined with zipfile or docfile.", + } + ) + try: + validate_component_snapshot_paths(source_component) + except ValueError as error: + raise serializers.ValidationError( + {"from_component": ("Source component contains unsafe file paths.")} + ) from error + self.validate_from_component_overrides(attrs, source_component) + attrs = self.apply_from_component_defaults(attrs, source_component) + # Build new or patched Component instance with changed attributes if self.instance: instance = copy(self.instance) @@ -975,15 +1262,40 @@ def validate(self, attrs): ) from error # Call model validation here, DRF does not do that - instance.clean() + if source_component is not None and "repo" not in self.initial_data: + self.validate_local_from_component_instance(instance) + else: + instance.clean() - if not self.instance and not disable_autoshare: + if not self.instance and not disable_autoshare and source_component is None: repo = instance.suggest_repo_link() if repo: attrs["repo"] = instance.repo = repo attrs["branch"] = instance.branch = "" + if source_component is not None: + attrs["from_component"] = source_component return attrs + def create(self, validated_data): + source_component = validated_data.pop("from_component", None) + if source_component is None: + return super().create(validated_data) + + if "repo" not in self.initial_data: + validated_data["repo"] = "local:" + validated_data["vcs"] = "local" + for field in ("branch", "push", "push_branch"): + validated_data.pop(field, None) + + component = Component(**validated_data) + component.prepare_seed_from_component( + source_component.pk, + copy_addons=True, + seed_author=self.context["request"].user.get_author_name(), + ) + component.save(force_insert=True) + return component + class NotificationSerializer(serializers.ModelSerializer[Subscription]): project = ProjectSerializer(read_only=True) @@ -1129,6 +1441,65 @@ class LockRequestSerializer(ReadOnlySerializer): lock = serializers.BooleanField() +class TranslationCreateSerializer(ReadOnlySerializer): + language_code = serializers.CharField() + from_component = ComponentReferenceListField(required=False) + + def validate(self, attrs): + component = self.context["component"] + request = self.context["request"] + source_components = [] + for reference in attrs.get("from_component", []): + source_component = resolve_component_reference_with_error( + Component.objects.filter_access(request.user), + reference, + "from_component", + ) + if not request.user.has_perm("component.edit", source_component): + raise serializers.ValidationError( + { + "from_component": "You do not have permission to use this component." + } + ) + if source_component.source_language_id != component.source_language_id: + raise serializers.ValidationError( + { + "from_component": ( + "Source component needs to have same source language as target one." + ) + } + ) + if ( + source_component.project_id != component.project_id + and not source_component.project.contribute_shared_tm + ): + raise serializers.ValidationError( + { + "from_component": ( + "Project has disabled contribution to shared translation memory." + ) + } + ) + source_components.append(source_component) + + if source_components and not any( + source_component.translation_set.filter( + language__code=attrs["language_code"] + ).exists() + for source_component in source_components + ): + raise serializers.ValidationError( + { + "from_component": ( + "None of the referenced components contain the requested language." + ) + } + ) + + attrs["from_component"] = source_components + return attrs + + class UploadRequestSerializer(ReadOnlySerializer): file = serializers.FileField() author_email = serializers.EmailField(required=False) diff --git a/weblate/api/tests.py b/weblate/api/tests.py index 4e791b7544a6..67d94e7518ef 100644 --- a/weblate/api/tests.py +++ b/weblate/api/tests.py @@ -15,8 +15,9 @@ import responses from django.conf import settings from django.core.cache import cache +from django.core.exceptions import ValidationError from django.core.files import File -from django.test.utils import modify_settings +from django.test.utils import modify_settings, override_settings from django.urls import reverse from rest_framework.test import APITestCase from weblate_language_data.languages import LANGUAGES @@ -34,6 +35,11 @@ from weblate.memory.models import Memory from weblate.screenshots.models import Screenshot from weblate.trans.actions import ActionEvents +from weblate.trans.component_seed import ( + resolve_destination_snapshot_path, + resolve_source_snapshot_path, + validate_snapshot_relative_path, +) from weblate.trans.exceptions import FailedCommitError, FileParseError from weblate.trans.models import ( Category, @@ -62,6 +68,7 @@ STATE_NEEDS_REWRITING, STATE_TRANSLATED, ) +from weblate.vcs.base import Repository TEST_PO = get_test_file("cs.po") TEST_POT = get_test_file("hello-charset.pot") @@ -169,6 +176,55 @@ def grant_perm_to_user( self.user.groups.add(group) +class ComponentSeedPathTest(APITestCase): + def test_validate_snapshot_relative_path_rejects_escape(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + outside = os.path.join(tempdir, "..", "other", "evil.po") + with self.assertRaises(ValueError): + validate_snapshot_relative_path(tempdir, "../evil.po") + + with self.assertRaises(ValueError): + validate_snapshot_relative_path(tempdir, outside) + + def test_resolve_source_snapshot_path_rejects_restricted_source(self) -> None: + def reject_path(_path: str) -> None: + msg = "restricted path" + raise ValidationError(msg) + + with tempfile.TemporaryDirectory() as tempdir: + component = SimpleNamespace( + full_path=tempdir, + check_file_is_valid=reject_path, + repository=SimpleNamespace(resolve_symlinks=lambda path: path), + ) + + with self.assertRaises(ValueError): + resolve_source_snapshot_path(component, ".git/config") + + def test_resolve_destination_snapshot_path_rejects_symlink_escape(self) -> None: + with ( + tempfile.TemporaryDirectory() as tempdir, + tempfile.TemporaryDirectory() as outside, + ): + os.makedirs(os.path.join(tempdir, "metadata"), exist_ok=True) + os.symlink(outside, os.path.join(tempdir, "metadata", "cs")) + + class DummyRepository: + def __init__(self, path: str) -> None: + self.path = path + + def resolve_symlinks(self, path: str) -> str: + return Repository.resolve_symlinks(self, path) + + component = SimpleNamespace( + full_path=tempdir, + repository=DummyRepository(tempdir), + ) + + with self.assertRaises(ValueError): + resolve_destination_snapshot_path(component, "metadata/cs/title.txt") + + class UserAPITest(APIBaseTest): def test_list(self) -> None: response = self.client.get(reverse("api:user-list")) @@ -4065,6 +4121,540 @@ def test_delete(self) -> None: ) self.assertEqual(Component.objects.count(), 1) + def test_create_component_from_component_id(self) -> None: + translation = self.component.translation_set.get(language_code="cs") + unit = translation.unit_set.get(source="Hello, world!\n") + unit.translate(self.user, "Duplicated from source!\n", STATE_TRANSLATED) + self.component.commit_message = "Commit from source" + self.component.priority = 175 + self.component.save() + self.component.addon_set.create(name="weblate.gettext.linguas") + + self.do_request( + "api:project-components", + self.project_kwargs, + method="post", + code=201, + superuser=True, + format="json", + request={ + "name": "API copy", + "slug": "api-copy", + "priority": 120, + "from_component": self.component.pk, + }, + ) + + duplicate = Component.objects.get(slug="api-copy", project__slug="test") + self.assertEqual(duplicate.repo, "local:") + self.assertEqual(duplicate.vcs, "local") + self.assertEqual(duplicate.filemask, self.component.filemask) + self.assertEqual(duplicate.file_format, self.component.file_format) + self.assertEqual(duplicate.new_lang, self.component.new_lang) + self.assertEqual(duplicate.commit_message, "Commit from source") + self.assertEqual(duplicate.priority, 120) + self.assertTrue( + duplicate.addon_set.filter(name="weblate.gettext.linguas").exists() + ) + duplicated_translation = duplicate.translation_set.get(language_code="cs") + duplicated_unit = duplicated_translation.unit_set.get(source="Hello, world!\n") + self.assertEqual(duplicated_unit.target, "Duplicated from source!\n") + + def test_create_component_from_component_path(self) -> None: + category = self.component.project.category_set.create( + name="Category", slug="category" + ) + source = self.create_po( + name="source-category", project=self.component.project, category=category + ) + source.commit_message = "Path copy commit" + source.save() + + self.do_request( + "api:project-components", + self.project_kwargs, + method="post", + code=201, + superuser=True, + format="json", + request={ + "name": "API copy path", + "slug": "api-copy-path", + "from_component": source.full_slug, + }, + ) + + duplicate = Component.objects.get(slug="api-copy-path", project__slug="test") + self.assertEqual(duplicate.category, category) + self.assertEqual(duplicate.repo, "local:") + self.assertEqual(duplicate.vcs, "local") + self.assertEqual(duplicate.filemask, source.filemask) + self.assertEqual(duplicate.file_format, source.file_format) + self.assertEqual(duplicate.commit_message, "Path copy commit") + + def test_create_component_from_component_with_explicit_repo_skips_branch_defaults( + self, + ) -> None: + Component.objects.filter(pk=self.component.pk).update( + branch="source-branch", + push="origin-source", + push_branch="push-source", + ) + self.component.refresh_from_db() + other = self.create_po(name="other-target", project=self.component.project) + + self.do_request( + "api:project-components", + self.project_kwargs, + method="post", + code=201, + superuser=True, + format="json", + request={ + "name": "API copy explicit repo", + "slug": "api-copy-explicit-repo", + "from_component": self.component.pk, + "repo": other.repo, + }, + ) + + duplicate = Component.objects.get( + slug="api-copy-explicit-repo", project__slug="test" + ) + self.assertEqual(duplicate.repo, other.repo) + self.assertEqual(duplicate.vcs, self.component.vcs) + self.assertNotEqual(duplicate.branch, "source-branch") + self.assertEqual(duplicate.push, "") + self.assertEqual(duplicate.push_branch, "") + + def test_create_component_from_component_rejects_layout_override(self) -> None: + response = self.do_request( + "api:project-components", + self.project_kwargs, + method="post", + code=400, + superuser=True, + format="json", + request={ + "name": "API copy invalid override", + "slug": "api-copy-invalid-override", + "from_component": self.component.pk, + "filemask": "translations/*.po", + }, + ) + + self.assertEqual( + response.data, + { + "type": "validation_error", + "errors": [ + { + "code": "invalid", + "detail": ( + "This field can not be overridden when using " + "from_component." + ), + "attr": "filemask", + } + ], + }, + ) + + def test_create_component_from_component_rejects_same_repo_branch(self) -> None: + source = self._create_component( + "po", + "translations/*.po", + name="source-same-branch", + project=self.component.project, + branch="translations", + ) + + response = self.do_request( + "api:project-components", + self.project_kwargs, + method="post", + code=400, + superuser=True, + format="json", + request={ + "name": "API copy same branch", + "slug": "api-copy-same-branch", + "from_component": source.pk, + "repo": source.repo, + "branch": source.branch, + }, + ) + + self.assertEqual( + response.data, + { + "type": "validation_error", + "errors": [ + { + "code": "invalid", + "detail": "Cloning into the same repository branch is not allowed.", + "attr": "branch", + } + ], + }, + ) + + def test_create_component_from_component_rejects_same_repo_default_branch( + self, + ) -> None: + response = self.do_request( + "api:project-components", + self.project_kwargs, + method="post", + code=400, + superuser=True, + format="json", + request={ + "name": "API copy same default branch", + "slug": "api-copy-same-default-branch", + "from_component": self.component.pk, + "repo": self.component.repo, + }, + ) + + self.assertEqual( + response.data, + { + "type": "validation_error", + "errors": [ + { + "code": "invalid", + "detail": "Cloning into the same repository branch is not allowed.", + "attr": "branch", + } + ], + }, + ) + + def test_create_component_from_component_rejects_branch_without_repo(self) -> None: + response = self.do_request( + "api:project-components", + self.project_kwargs, + method="post", + code=400, + superuser=True, + format="json", + request={ + "name": "API copy invalid branch", + "slug": "api-copy-invalid-branch", + "from_component": self.component.pk, + "branch": "release", + }, + ) + + self.assertEqual( + response.data, + { + "type": "validation_error", + "errors": [ + { + "code": "invalid", + "detail": "This field requires repo when using from_component.", + "attr": "branch", + } + ], + }, + ) + + def test_create_component_from_component_without_repo_skips_source_repo_validation( + self, + ) -> None: + Component.objects.filter(pk=self.component.pk).update( + repo="https://example.invalid/missing.git" + ) + self.component.refresh_from_db() + + self.do_request( + "api:project-components", + self.project_kwargs, + method="post", + code=201, + superuser=True, + format="json", + request={ + "name": "API copy local snapshot", + "slug": "api-copy-local-snapshot", + "from_component": self.component.pk, + }, + ) + + duplicate = Component.objects.get( + slug="api-copy-local-snapshot", project__slug=self.component.project.slug + ) + self.assertEqual(duplicate.repo, "local:") + self.assertTrue(duplicate.translation_set.filter(language_code="cs").exists()) + + def test_create_component_from_component_appstore(self) -> None: + source = self.create_appstore( + name="source-appstore", project=self.component.project + ) + source_translation = source.translation_set.get(language_code="cs") + source_unit = source_translation.unit_set.get( + source="Weblate - continuous localization" + ) + source_unit.translate( + self.user, "Weblate - metadata duplicate", STATE_TRANSLATED + ) + + self.do_request( + "api:project-components", + self.project_kwargs, + method="post", + code=201, + superuser=True, + format="json", + request={ + "name": "API copy appstore", + "slug": "api-copy-appstore", + "from_component": source.pk, + }, + ) + + duplicate = Component.objects.get( + slug="api-copy-appstore", project__slug=self.component.project.slug + ) + duplicated_translation = duplicate.translation_set.get(language_code="cs") + duplicated_unit = duplicated_translation.unit_set.get( + source="Weblate - continuous localization" + ) + self.assertEqual(duplicated_unit.target, "Weblate - metadata duplicate") + self.assertGreater(len(source_translation.filenames), 1) + duplicate_files = { + os.path.relpath(filename, duplicate.full_path) + for filename in duplicated_translation.filenames + } + self.assertTrue( + { + os.path.relpath(filename, source.full_path) + for filename in source_translation.filenames + }.issubset(duplicate_files) + ) + self.assertIn("metadata/cs/title.txt", duplicate_files) + + def test_create_component_from_component_does_not_clone_addons_cross_project( + self, + ) -> None: + source_project = self.create_project(name="Source", slug="source") + source = self.create_po(name="source-copy", project=source_project) + source.addon_set.create( + name="weblate.gettext.linguas", + configuration={"secret": "cross-project"}, + ) + + self.do_request( + "api:project-components", + self.project_kwargs, + method="post", + code=201, + superuser=True, + format="json", + request={ + "name": "API copy no addons", + "slug": "api-copy-no-addons", + "from_component": source.pk, + }, + ) + + duplicate = Component.objects.get( + slug="api-copy-no-addons", project__slug=self.component.project.slug + ) + self.assertFalse( + duplicate.addon_set.filter(name="weblate.gettext.linguas").exists() + ) + + @override_settings(CELERY_TASK_ALWAYS_EAGER=False) + def test_create_component_from_component_queues_seed(self) -> None: + with ( + patch( + "weblate.trans.tasks.component_after_save.delay", + return_value=SimpleNamespace(id="component-task"), + ) as delay, + patch( + "weblate.trans.models.component.AsyncResult", + return_value=SimpleNamespace(id="component-task", ready=lambda: False), + ), + ): + self.do_request( + "api:project-components", + self.project_kwargs, + method="post", + code=201, + superuser=True, + format="json", + request={ + "name": "API copy async", + "slug": "api-copy-async", + "from_component": self.component.pk, + }, + ) + + self.assertEqual(delay.call_count, 1) + self.assertEqual( + delay.call_args.args[0], + Component.objects.get(slug="api-copy-async").pk, + ) + self.assertEqual( + delay.call_args.kwargs["seed_source_component_id"], self.component.pk + ) + self.assertTrue(delay.call_args.kwargs["copy_seed_addons"]) + self.assertEqual( + delay.call_args.kwargs["seed_author"], self.user.get_author_name() + ) + self.assertTrue(delay.call_args.kwargs["skip_push"]) + + def test_create_component_from_component_seed_uses_skip_push(self) -> None: + duplicate = Component.objects.create( + project=self.component.project, + name="API copy seeded", + slug="api-copy-seeded", + vcs="local", + repo="local:", + filemask=self.component.filemask, + file_format=self.component.file_format, + ) + + with patch( + "weblate.trans.component_seed.seed_component_from_source" + ) as seed_component_from_source: + duplicate.after_save( + changed_git=False, + changed_setup=False, + changed_template=False, + changed_variant=False, + changed_enforced_checks=False, + skip_push=True, + create=True, + seed_source_component_id=self.component.pk, + copy_seed_addons=False, + seed_author=self.user.get_author_name(), + ) + + seed_component_from_source.assert_called_once_with( + duplicate, + self.component, + author_name=self.user.get_author_name(), + skip_push=True, + ) + + def test_create_component_from_component_falls_back_to_scan_when_seed_empty( + self, + ) -> None: + duplicate = Component.objects.create( + project=self.component.project, + name="API copy empty seed", + slug="api-copy-empty-seed", + vcs="local", + repo="local:", + filemask=self.component.filemask, + file_format=self.component.file_format, + ) + + with ( + patch.object(duplicate, "create_translations") as create_translations, + patch( + "weblate.trans.component_seed.seed_component_from_source", + return_value=False, + ) as seed_component_from_source, + ): + duplicate.after_save( + changed_git=False, + changed_setup=True, + changed_template=False, + changed_variant=False, + changed_enforced_checks=False, + skip_push=True, + create=True, + seed_source_component_id=self.component.pk, + copy_seed_addons=False, + seed_author=self.user.get_author_name(), + ) + + create_translations.assert_called_once_with(force=True, changed_template=False) + seed_component_from_source.assert_called_once_with( + duplicate, + self.component, + author_name=self.user.get_author_name(), + skip_push=True, + ) + + def test_create_component_from_component_skips_initial_translation_scan( + self, + ) -> None: + duplicate = Component.objects.create( + project=self.component.project, + name="API copy one scan", + slug="api-copy-one-scan", + vcs="local", + repo="local:", + filemask=self.component.filemask, + file_format=self.component.file_format, + ) + + with ( + patch.object(duplicate, "create_translations") as create_translations, + patch( + "weblate.trans.component_seed.seed_component_from_source" + ) as seed_component_from_source, + ): + duplicate.after_save( + changed_git=False, + changed_setup=True, + changed_template=False, + changed_variant=False, + changed_enforced_checks=False, + skip_push=True, + create=True, + seed_source_component_id=self.component.pk, + copy_seed_addons=False, + seed_author=self.user.get_author_name(), + ) + + create_translations.assert_not_called() + seed_component_from_source.assert_called_once_with( + duplicate, + self.component, + author_name=self.user.get_author_name(), + skip_push=True, + ) + + def test_create_component_from_component_rejects_unsafe_snapshot_path(self) -> None: + with patch( + "weblate.api.serializers.validate_component_snapshot_paths", + side_effect=ValueError("../evil.po"), + ): + response = self.do_request( + "api:project-components", + self.project_kwargs, + method="post", + code=400, + superuser=True, + format="json", + request={ + "name": "API copy unsafe", + "slug": "api-copy-unsafe", + "from_component": self.component.pk, + }, + ) + + self.assertEqual( + response.data, + { + "type": "validation_error", + "errors": [ + { + "code": "invalid", + "detail": "Source component contains unsafe file paths.", + "attr": "from_component", + } + ], + }, + ) + def test_create_translation(self) -> None: self.component.new_lang = "add" self.component.new_base = "po/hello.pot" @@ -4089,6 +4679,166 @@ def test_create_translation_existing(self) -> None: request={"language_code": "cs"}, ) + def test_create_translation_from_component(self) -> None: + target = self.create_po_new_base(name="target", project=self.component.project) + source_project_one = self.create_project( + name="Source one", slug="source-one", contribute_shared_tm=True + ) + source_project_two = self.create_project( + name="Source two", slug="source-two", contribute_shared_tm=True + ) + source_one = self.create_po_new_base( + name="source-one", project=source_project_one + ) + source_two = self.create_po_new_base( + name="source-two", project=source_project_two + ) + language = Language.objects.get(code="fa") + + for component, text in ( + (source_one, "First source translation!\n"), + (source_two, "Second source translation!\n"), + ): + translation = component.add_new_language(language, None) + self.assertIsNotNone(translation) + if component == source_one: + unit = translation.unit_set.get(source="Hello, world!\n") + else: + unit = next( + candidate + for candidate in translation.unit_set.exclude( + source="Hello, world!\n" + ) + if not candidate.is_plural + ) + unit.translate(self.user, text, STATE_TRANSLATED) + if component == source_two: + fallback_source = unit.source + + self.do_request( + "api:component-translations", + {"project__slug": target.project.slug, "slug": target.slug}, + method="post", + code=201, + superuser=True, + format="json", + request={ + "language_code": "fa", + "from_component": [source_one.full_slug, str(source_two.pk)], + }, + ) + + created = target.translation_set.get(language_code="fa") + first_unit = created.unit_set.get(source="Hello, world!\n") + second_unit = created.unit_set.get(source=fallback_source) + self.assertEqual(first_unit.target, "First source translation!\n") + self.assertEqual(second_unit.target.rstrip("\n"), "Second source translation!") + + def test_create_translation_from_component_duplicates(self) -> None: + target = self.create_po_new_base(name="target", project=self.component.project) + source = self.create_po_new_base(name="source", project=self.component.project) + language = Language.objects.get(code="fa") + + translation = source.add_new_language(language, None) + self.assertIsNotNone(translation) + unit = translation.unit_set.get(source="Hello, world!\n") + unit.translate(self.user, "Duplicated source translation!\n", STATE_TRANSLATED) + + self.do_request( + "api:component-translations", + {"project__slug": target.project.slug, "slug": target.slug}, + method="post", + code=201, + superuser=True, + format="json", + request={ + "language_code": "fa", + "from_component": [source.full_slug, source.full_slug], + }, + ) + + created = target.translation_set.get(language_code="fa") + created_unit = created.unit_set.get(source="Hello, world!\n") + self.assertEqual(created_unit.target, "Duplicated source translation!\n") + + def test_create_translation_from_component_language_code_style(self) -> None: + target = self.create_po_new_base(name="target", project=self.component.project) + source = self.create_po_new_base( + name="source-style", + project=self.component.project, + language_code_style="bcp", + ) + language = Language.objects.get(code="pt_BR") + + translation = source.add_new_language(language, None) + self.assertIsNotNone(translation) + self.assertEqual(translation.language.code, "pt_BR") + self.assertEqual(translation.language_code, "pt-BR") + unit = translation.unit_set.get(source="Hello, world!\n") + unit.translate(self.user, "Styled source translation!\n", STATE_TRANSLATED) + + self.do_request( + "api:component-translations", + {"project__slug": target.project.slug, "slug": target.slug}, + method="post", + code=201, + superuser=True, + format="json", + request={ + "language_code": "pt_BR", + "from_component": [source.full_slug], + }, + ) + + created = target.translation_set.get(language__code="pt_BR") + created_unit = created.unit_set.get(source="Hello, world!\n") + self.assertEqual(created_unit.target, "Styled source translation!\n") + + def test_create_translation_from_component_requires_edit_permission(self) -> None: + target = self.create_po_new_base(name="target", project=self.component.project) + source = self.create_po_new_base(name="source", project=self.component.project) + language = Language.objects.get(code="fa") + source_translation = source.add_new_language(language, None) + self.assertIsNotNone(source_translation) + + self.do_request( + "api:component-translations", + {"project__slug": target.project.slug, "slug": target.slug}, + method="post", + code=400, + format="json", + request={ + "language_code": "fa", + "from_component": [source.full_slug], + }, + ) + + def test_create_translation_from_component_validation_has_no_side_effects( + self, + ) -> None: + target = self.create_po_new_base(name="target", project=self.component.project) + source_project = self.create_project( + name="Source", slug="source", contribute_shared_tm=False + ) + source = self.create_po_new_base(name="source", project=source_project) + language = Language.objects.get(code="fa") + source_translation = source.add_new_language(language, None) + self.assertIsNotNone(source_translation) + + self.do_request( + "api:component-translations", + {"project__slug": target.project.slug, "slug": target.slug}, + method="post", + code=400, + superuser=True, + format="json", + request={ + "language_code": "fa", + "from_component": [source.full_slug], + }, + ) + self.assertFalse(target.translation_set.filter(language_code="fa").exists()) + def test_create_translation_invalid_language_code(self) -> None: self.component.new_lang = "add" self.component.new_base = "po/hello.pot" @@ -5194,6 +5944,22 @@ def test_autotranslate(self, format: str = "multipart") -> None: # noqa: A002 code=200, ) self.assertContains(response, "Automatic translation completed") + response = self.do_request( + "api:translation-autotranslate", + self.translation_kwargs, + superuser=True, + method="post", + request={ + "mode": "suggest", + "q": "state: None: self.user.profile.increase_count("translated", self.updated) @transaction.atomic - def process_others(self, source_component_id: int | None) -> None: + def process_others(self, source_component_ids: list[int] | None) -> None: """Perform automatic translation based on other components.""" sources = Unit.objects.filter( translation__plural=self.translation.plural, state__gte=STATE_TRANSLATED, ) source_language = self.translation.component.source_language - if source_component_id: - component = Component.objects.get(id=source_component_id) - - if ( - not component.project.contribute_shared_tm - and component.project != self.translation.component.project - ): - msg = "Project has disabled contribution to shared translation memory." - raise PermissionDenied(msg) - if component.source_language != source_language: - msg = "Component have different source languages." - raise PermissionDenied(msg) - sources = sources.filter(translation__component=component) + component_ids = list(dict.fromkeys(source_component_ids or [])) + if component_ids: + components = list(Component.objects.filter(id__in=component_ids)) + component_map = {component.id: component for component in components} + if len(component_map) != len(component_ids): + raise Component.DoesNotExist + + for component_id in component_ids: + component = component_map[component_id] + if ( + not component.project.contribute_shared_tm + and component.project != self.translation.component.project + ): + msg = "Project has disabled contribution to shared translation memory." + raise PermissionDenied(msg) + if component.source_language != source_language: + msg = "Component have different source languages." + raise PermissionDenied(msg) + sources = sources.filter(translation__component_id__in=component_ids) else: project = self.translation.component.project sources = sources.filter( @@ -193,12 +199,26 @@ def process_others(self, source_component_id: int | None) -> None: ) # Fetch available translations - translations = { - source: split_plural(target) - for source, target in sources.filter( - source__lower__md5__in=source_md5s - ).values_list("source", "target") - } + translations: dict[str, list[str]] = {} + filtered_sources = sources.filter(source__lower__md5__in=source_md5s) + if component_ids: + component_priority = { + component_id: index for index, component_id in enumerate(component_ids) + } + translation_priority: dict[str, int] = {} + for component_id, source, target in filtered_sources.values_list( + "translation__component_id", "source", "target" + ): + priority = component_priority[component_id] + if priority >= translation_priority.get(source, len(component_ids)): + continue + translations[source] = split_plural(target) + translation_priority[source] = priority + else: + translations = { + source: split_plural(target) + for source, target in filtered_sources.values_list("source", "target") + } # Fetch translated unit IDs # Cannot use get_units() directly as SELECT FOR UPDATE cannot be used with JOIN @@ -330,7 +350,7 @@ def perform( auto_source: Literal["mt", "others"], engines: list[str], threshold: int, - source_component_id: int | None, + source_component_ids: list[int] | None, ) -> str: translation = self.translation translation.log_info( @@ -338,13 +358,15 @@ def perform( self.mode, current_task.request.id if current_task and current_task.request.id else "", auto_source, - ", ".join(engines) if engines else source_component_id, + ", ".join(engines) + if engines + else ", ".join(str(item) for item in source_component_ids or []), ) try: if auto_source == "mt": self.process_mt(engines, threshold) else: - self.process_others(source_component_id) + self.process_others(source_component_ids) except (MachineTranslationError, Component.DoesNotExist) as error: translation.log_error("failed automatic translation: %s", error) return gettext("Automatic translation failed: %s") % error @@ -408,7 +430,7 @@ def perform( auto_source: Literal["mt", "others"], engines: list[str], threshold: int, - source_component_id: int | None, + source_component_ids: list[int] | None, ) -> str: for pos, translation in enumerate(self.translations, start=1): auto_translate = AutoTranslate( @@ -424,7 +446,7 @@ def perform( auto_source=auto_source, engines=engines, threshold=threshold, - source_component_id=source_component_id, + source_component_ids=source_component_ids, ) self.updated += auto_translate.updated self.set_progress(pos) diff --git a/weblate/trans/component_seed.py b/weblate/trans/component_seed.py new file mode 100644 index 000000000000..e8a22ddf0755 --- /dev/null +++ b/weblate/trans/component_seed.py @@ -0,0 +1,248 @@ +# Copyright © Michal Čihař +# +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import annotations + +import os +import shutil +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING + +from django.core.exceptions import ValidationError + +from weblate.addons.models import Addon +from weblate.formats.base import UnitNotFoundError + +if TYPE_CHECKING: + from weblate.trans.models import Component, Translation, Unit + + +def validate_snapshot_relative_path(base_path: str, path: str) -> str: + """Return a safe relative path rooted under base_path.""" + if os.path.isabs(path): + absolute_path = os.path.abspath(path) + base_path = os.path.abspath(base_path) + try: + common_path = os.path.commonpath([base_path, absolute_path]) + except ValueError as error: + raise ValueError(path) from error + if common_path != base_path: + raise ValueError(path) + relative_path = os.path.relpath(absolute_path, base_path) + else: + relative_path = path + + normalized = os.path.normpath(relative_path) + if normalized in {"", "."}: + raise ValueError(path) + + parts = Path(relative_path).parts + if os.path.isabs(relative_path) or any(part == ".." for part in parts): + raise ValueError(path) + + normalized_parts = Path(normalized).parts + if any(part in {"", "."} for part in normalized_parts): + raise ValueError(path) + + return normalized + + +def resolve_source_snapshot_path(component: Component, path: str) -> str: + """Resolve a source repository path using repository safety checks.""" + if os.path.isabs(path): + try: + relative_path = os.path.relpath(path, component.full_path) + except ValueError as error: + raise ValueError(path) from error + else: + relative_path = path + + relative_path = validate_snapshot_relative_path(component.full_path, relative_path) + + try: + component.check_file_is_valid(relative_path) + resolved_path = component.repository.resolve_symlinks(relative_path) + except ValidationError as error: + raise ValueError(path) from error + + return validate_snapshot_relative_path(component.full_path, resolved_path) + + +def resolve_destination_snapshot_path(component: Component, path: str) -> str: + """Resolve a destination repository path using repository safety checks.""" + relative_path = validate_snapshot_relative_path(component.full_path, path) + + try: + resolved_path = component.repository.resolve_symlinks(relative_path) + except ValueError as error: + raise ValueError(path) from error + + return os.path.join( + component.full_path, + validate_snapshot_relative_path(component.full_path, resolved_path), + ) + + +def validate_component_snapshot_paths(source_component: Component) -> None: + """Ensure all source component snapshot paths are safe.""" + for translation in source_component.translation_set.order_by("id"): + if translation.filename: + resolve_source_snapshot_path(source_component, translation.filename) + for filename in translation.filenames: + resolve_source_snapshot_path(source_component, filename) + + for relative_name in ( + source_component.template, + source_component.intermediate, + source_component.new_base, + ): + if relative_name: + resolve_source_snapshot_path(source_component, relative_name) + + +def apply_unit_state_to_store(store, unit: Unit) -> None: + """Overlay unit content from DB state onto a translation store.""" + try: + ttunit, add = store.find_unit(unit.context, unit.source) + if add: + store.add_unit(ttunit) + except UnitNotFoundError: + ttunit = store.new_unit( + unit.context, unit.get_source_plurals(), unit.get_target_plurals() + ) + + if unit.is_plural: + ttunit.set_target(unit.get_target_plurals()) + else: + ttunit.set_target(unit.target) + ttunit.set_explanation(unit.explanation) + ttunit.set_source_explanation(unit.source_unit.explanation) + ttunit.set_state(unit.state) + ttunit.set_automatically_translated(unit.automatically_translated) + + +def serialize_translation_snapshot(translation: Translation) -> dict[str, bytes]: + """ + Serialize translation content based on current DB unit state. + + This keeps component duplication independent from commits in the source repository. + """ + if not translation.filename: + return {} + + main_filename = translation.get_filename() + if main_filename is None: + return {} + + main_relative_name = resolve_source_snapshot_path( + translation.component, translation.filename + ) + relative_files = { + resolve_source_snapshot_path(translation.component, filename): filename + for filename in translation.filenames + } + + with tempfile.TemporaryDirectory() as tempdir: + for relative_name, source_name in relative_files.items(): + target_name = os.path.join(tempdir, relative_name) + os.makedirs(os.path.dirname(target_name), exist_ok=True) + shutil.copy2(source_name, target_name) + + if translation.is_template: + return { + relative_name: Path(os.path.join(tempdir, relative_name)).read_bytes() + for relative_name in relative_files + } + + temp_filename = os.path.join(tempdir, main_relative_name) + store = translation.load_store(temp_filename) + for unit in translation.unit_set.prefetch_full().order_by("position"): + apply_unit_state_to_store(store, unit) + store.save() + + result: dict[str, bytes] = {} + for filename in store.get_filenames(): + relative_name = validate_snapshot_relative_path(tempdir, filename) + snapshot_name = os.path.join(tempdir, relative_name) + if not os.path.exists(snapshot_name): + continue + result[relative_name] = Path(snapshot_name).read_bytes() + + return result + + +def build_component_snapshot(source_component: Component) -> dict[str, bytes]: + """Build a source snapshot for component duplication.""" + validate_component_snapshot_paths(source_component) + files: dict[str, bytes] = {} + + for translation in source_component.translation_set.order_by("id"): + files.update(serialize_translation_snapshot(translation)) + + for relative_name in ( + source_component.template, + source_component.intermediate, + source_component.new_base, + ): + if not relative_name: + continue + safe_relative_name = resolve_source_snapshot_path( + source_component, relative_name + ) + absolute_name = os.path.join(source_component.full_path, safe_relative_name) + if os.path.isdir(absolute_name): + continue + if os.path.exists(absolute_name) and safe_relative_name not in files: + files[safe_relative_name] = Path(absolute_name).read_bytes() + + return files + + +def seed_component_from_source( + component: Component, + source_component: Component, + *, + author_name: str, + skip_push: bool, +) -> bool: + """Seed a newly created component from another component.""" + snapshot = build_component_snapshot(source_component) + if not snapshot: + return False + + component.configure_branch() + + written_files: list[str] = [] + with component.repository.lock: + for relative_name, content in snapshot.items(): + target_name = resolve_destination_snapshot_path(component, relative_name) + os.makedirs(os.path.dirname(target_name), exist_ok=True) + Path(target_name).write_bytes(content) + written_files.append(target_name) + + component.commit_files( + author=author_name, + files=written_files, + message=(f"Seed translations from {source_component.full_slug}"), + store_hash=False, + skip_push=skip_push, + ) + + component.create_translations(force=True) + return True + + +def clone_component_addons(component: Component, source_component: Component) -> None: + """Clone component-scoped add-ons from the source component.""" + if component.project_id != source_component.project_id: + return + + addons = Addon.objects.filter(component=source_component, repo_scope=False) + for addon in addons: + if component.addon_set.filter(name=addon.name).exists(): + continue + if not addon.addon.can_install(component=component): + continue + addon.addon.create(component=component, configuration=addon.configuration) diff --git a/weblate/trans/management/commands/auto_translate.py b/weblate/trans/management/commands/auto_translate.py index bdd7a4dafcfd..063cdbc20d83 100644 --- a/weblate/trans/management/commands/auto_translate.py +++ b/weblate/trans/management/commands/auto_translate.py @@ -110,7 +110,9 @@ def handle(self, *args, **options) -> None: message = auto.perform( auto_source="mt" if options["mt"] else "others", - source_component_id=source_component_id, + source_component_ids=( + [source_component_id] if source_component_id is not None else None + ), engines=options["mt"], threshold=options["threshold"], ) diff --git a/weblate/trans/models/component.py b/weblate/trans/models/component.py index b92175aea2bd..fb036645a545 100644 --- a/weblate/trans/models/component.py +++ b/weblate/trans/models/component.py @@ -975,6 +975,10 @@ def save(self, *args, **kwargs) -> None: """ from weblate.trans.tasks import component_after_save + seed_source_component_id = getattr(self, "seed_source_component_id", None) + copy_seed_addons = getattr(self, "copy_seed_addons", False) + seed_author = getattr(self, "seed_author", None) + self.drop_file_format_cache() self.set_default_branch() @@ -1079,6 +1083,9 @@ def save(self, *args, **kwargs) -> None: changed_enforced_checks=changed_enforced_checks, skip_push=kwargs.get("force_insert", False), create=create, + seed_source_component_id=seed_source_component_id, + copy_seed_addons=copy_seed_addons, + seed_author=seed_author, ) else: self.queue_background_task( @@ -1091,6 +1098,9 @@ def save(self, *args, **kwargs) -> None: changed_enforced_checks=changed_enforced_checks, skip_push=kwargs.get("force_insert", False), create=create, + seed_source_component_id=seed_source_component_id, + copy_seed_addons=copy_seed_addons, + seed_author=seed_author, ) if self.old_component_settings["check_flags"] != self.check_flags: @@ -1107,6 +1117,18 @@ def save(self, *args, **kwargs) -> None: if update_tm: import_memory.delay_on_commit(self.project.id, self.pk) + def prepare_seed_from_component( + self, + source_component_id: int, + *, + copy_addons: bool, + seed_author: str | None, + ) -> None: + """Store transient seed settings for processing in after_save.""" + self.seed_source_component_id = source_component_id + self.copy_seed_addons = copy_addons + self.seed_author = seed_author + @cached_property def cached_links(self) -> models.QuerySet[Project]: return self.links.all() @@ -3693,11 +3715,20 @@ def after_save( changed_enforced_checks: bool, skip_push: bool, create: bool, + seed_source_component_id: int | None = None, + copy_seed_addons: bool = False, + seed_author: str | None = None, ) -> None: + from weblate.trans.component_seed import ( + clone_component_addons, + seed_component_from_source, + ) + self.store_background_task() self.translations_progress = 0 self.translations_count = 0 self.progress_step(0) + has_seed_source = create and seed_source_component_id is not None # Configure git repo if there were changes if changed_git and (not create or not self.is_repo_link): # Bring VCS repo in sync with current model @@ -3709,7 +3740,9 @@ def after_save( # Rescan for possibly new translations if there were changes, needs to # be done after actual creating the object above was_change = False - if changed_setup: + if has_seed_source: + was_change = False + elif changed_setup: was_change = self.create_translations( force=True, changed_template=changed_template ) @@ -3720,6 +3753,23 @@ def after_save( if changed_variant and not was_change: self.update_variants() + if has_seed_source: + source_component = Component.objects.get(pk=seed_source_component_id) + seeded = seed_component_from_source( + self, + source_component, + author_name=seed_author or "Weblate ", + skip_push=skip_push, + ) + if not seeded: + was_change = self.create_seed_fallback_translations( + changed_setup=changed_setup, + changed_git=changed_git, + changed_template=changed_template, + ) + if copy_seed_addons: + clone_component_addons(self, source_component) + # Update changed enforced checks if changed_enforced_checks: self.update_enforced_checks() @@ -3733,6 +3783,7 @@ def after_save( # Update alerts after stats update self.update_alerts() + if self.linked_component: self.linked_component.update_alerts() @@ -3753,6 +3804,18 @@ def after_save( self.log_debug("triggering add-on: %s", addon.name) addon.addon.post_configure_run() + def create_seed_fallback_translations( + self, *, changed_setup: bool, changed_git: bool, changed_template: bool + ) -> bool: + """Run normal translation discovery when seeded creation has no snapshot.""" + if changed_setup: + return self.create_translations( + force=True, changed_template=changed_template + ) + if changed_git: + return self.create_translations() + return False + def update_variants(self, updated_units=None) -> None: component_units = Unit.objects.filter(translation__component=self, variant=None) diff --git a/weblate/trans/tasks.py b/weblate/trans/tasks.py index 4fe220dd4be8..ae6af5c7de3d 100644 --- a/weblate/trans/tasks.py +++ b/weblate/trans/tasks.py @@ -399,6 +399,7 @@ def component_alerts(component_ids=None) -> None: @transaction.atomic def component_after_save( pk: int, + *, changed_git: bool, changed_setup: bool, changed_template: bool, @@ -406,6 +407,9 @@ def component_after_save( changed_enforced_checks: bool, skip_push: bool, create: bool, + seed_source_component_id: int | None = None, + copy_seed_addons: bool = False, + seed_author: str | None = None, ) -> dict[Literal["component"], int]: component = Component.objects.get(pk=pk) component.after_save( @@ -416,6 +420,9 @@ def component_after_save( changed_enforced_checks=changed_enforced_checks, skip_push=skip_push, create=create, + seed_source_component_id=seed_source_component_id, + copy_seed_addons=copy_seed_addons, + seed_author=seed_author, ) return {"component": pk} @@ -630,7 +637,9 @@ def auto_translate( # noqa: PLR0913 auto_source=auto_source, engines=engines, threshold=threshold, - source_component_id=source_component_id, + source_component_ids=( + [source_component_id] if source_component_id is not None else None + ), ) except PermissionDenied as error: result.update({"message": str(error)}) @@ -666,7 +675,9 @@ def auto_translate_component( auto_source=auto_source, engines=engines, threshold=threshold, - source_component_id=source_component_id, + source_component_ids=( + [source_component_id] if source_component_id is not None else None + ), ) component_obj.update_source_checks() component_obj.run_batched_checks()