Source code for langchain_community.document_transformers.doctran_text_translate

from __future__ import annotations

import asyncio
from typing import Any, Optional, Sequence

from langchain_core.documents import BaseDocumentTransformer, Document
from langchain_core.runnables.config import run_in_executor
from langchain_core.utils import get_from_env



[docs]
class DoctranTextTranslator(BaseDocumentTransformer):
    """Translate text documents using doctran.

    Arguments:
        openai_api_key: OpenAI API key. Can also be specified via environment variable
        ``OPENAI_API_KEY``.
        language: The language to translate *to*.

    Example:
        .. code-block:: python

        from langchain_community.document_transformers import DoctranTextTranslator

        # Pass in openai_api_key or set env var OPENAI_API_KEY
        qa_translator = DoctranTextTranslator(language="spanish")
        translated_document = await qa_translator.atransform_documents(documents)
    """


[docs]
    def __init__(
        self,
        openai_api_key: Optional[str] = None,
        language: str = "english",
        openai_api_model: Optional[str] = None,
    ) -> None:
        self.openai_api_key = openai_api_key or get_from_env(
            "openai_api_key", "OPENAI_API_KEY"
        )
        self.openai_api_model = openai_api_model or get_from_env(
            "openai_api_model", "OPENAI_API_MODEL"
        )
        self.language = language


    async def _aparse_document(
        self, doctran: Any, index: int, doc: Document
    ) -> tuple[int, Any]:
        parsed_doc = await run_in_executor(
            None, doctran.parse, content=doc.page_content, metadata=doc.metadata
        )
        return index, parsed_doc

    async def _atranslate_document(
        self, index: int, doc: Any, language: str
    ) -> tuple[int, Any]:
        translated_doc = await run_in_executor(
            None, lambda: doc.translate(language=language).execute()
        )
        return index, translated_doc


[docs]
    async def atransform_documents(
        self, documents: Sequence[Document], **kwargs: Any
    ) -> Sequence[Document]:
        """Translates text documents using doctran."""
        try:
            from doctran import Doctran

            doctran = Doctran(
                openai_api_key=self.openai_api_key, openai_model=self.openai_api_model
            )
        except ImportError:
            raise ImportError(
                "Install doctran to use this parser. (pip install doctran)"
            )

        parse_tasks = [
            self._aparse_document(doctran, i, doc) for i, doc in enumerate(documents)
        ]
        parsed_results = await asyncio.gather(*parse_tasks)

        parsed_results.sort(key=lambda x: x[0])
        doctran_docs = [doc for _, doc in parsed_results]

        translate_tasks = [
            self._atranslate_document(i, doc, self.language)
            for i, doc in enumerate(doctran_docs)
        ]
        translated_results = await asyncio.gather(*translate_tasks)

        translated_results.sort(key=lambda x: x[0])
        translated_docs = [doc for _, doc in translated_results]

        return [
            Document(page_content=doc.transformed_content, metadata=doc.metadata)
            for doc in translated_docs
        ]



[docs]
    def transform_documents(
        self, documents: Sequence[Document], **kwargs: Any
    ) -> Sequence[Document]:
        """Translates text documents using doctran."""
        try:
            from doctran import Doctran

            doctran = Doctran(
                openai_api_key=self.openai_api_key, openai_model=self.openai_api_model
            )
        except ImportError:
            raise ImportError(
                "Install doctran to use this parser. (pip install doctran)"
            )
        doctran_docs = [
            doctran.parse(content=doc.page_content, metadata=doc.metadata)
            for doc in documents
        ]
        for i, doc in enumerate(doctran_docs):
            doctran_docs[i] = doc.translate(language=self.language).execute()
        return [
            Document(page_content=doc.transformed_content, metadata=doc.metadata)
            for doc in doctran_docs
        ]