Source code for langchain.evaluation.exact_match.base

import string
from typing import Any

from typing_extensions import override

from langchain.evaluation.schema import StringEvaluator



[docs]
class ExactMatchStringEvaluator(StringEvaluator):
    """Compute an exact match between the prediction and the reference.

    Examples
    ----------
    >>> evaluator = ExactMatchChain()
    >>> evaluator.evaluate_strings(
            prediction="Mindy is the CTO",
            reference="Mindy is the CTO",
        )  # This will return {'score': 1.0}

    >>> evaluator.evaluate_strings(
            prediction="Mindy is the CTO",
            reference="Mindy is the CEO",
        )  # This will return {'score': 0.0}
    """


[docs]
    def __init__(
        self,
        *,
        ignore_case: bool = False,
        ignore_punctuation: bool = False,
        ignore_numbers: bool = False,
        **_: Any,
    ):
        """Initialize the ExactMatchStringEvaluator.

        Args:
            ignore_case: Whether to ignore case when comparing strings.
                Defaults to False.
            ignore_punctuation: Whether to ignore punctuation when comparing strings.
                Defaults to False.
            ignore_numbers: Whether to ignore numbers when comparing strings.
                Defaults to False.
        """
        super().__init__()
        self.ignore_case = ignore_case
        self.ignore_punctuation = ignore_punctuation
        self.ignore_numbers = ignore_numbers


    @property
    def requires_input(self) -> bool:
        """
        This evaluator does not require input.
        """
        return False

    @property
    def requires_reference(self) -> bool:
        """
        This evaluator requires a reference.
        """
        return True

    @property
    def input_keys(self) -> list[str]:
        """
        Get the input keys.

        Returns:
            List[str]: The input keys.
        """
        return ["reference", "prediction"]

    @property
    def evaluation_name(self) -> str:
        """
        Get the evaluation name.

        Returns:
            str: The evaluation name.
        """
        return "exact_match"

    @override
    def _evaluate_strings(  # type: ignore[override]
        self,
        *,
        prediction: str,
        reference: str,
        **kwargs: Any,
    ) -> dict:
        """
        Evaluate the exact match between the prediction and the reference.

        Args:
            prediction (str): The prediction string.
            reference (Optional[str], optional): The reference string.

        Returns:
            dict: The evaluation results containing the score.
        """
        if self.ignore_case:
            prediction = prediction.lower()
            reference = reference.lower()
        if self.ignore_punctuation:
            prediction = prediction.translate(str.maketrans("", "", string.punctuation))
            reference = reference.translate(str.maketrans("", "", string.punctuation))
        if self.ignore_numbers:
            prediction = prediction.translate(str.maketrans("", "", string.digits))
            reference = reference.translate(str.maketrans("", "", string.digits))
        return {"score": int(prediction == reference)}