






'''simple docstring'''




import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    Pipeline,
    ZeroShotClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
__lowerCamelCase   :					List[str]   							=					{'''LayoutLMv2Config''', '''LayoutLMv3Config'''}







@is_pipeline_test
class    A_	(unittest.TestCase							):
       """simple docstring"""




       a__   		=	MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
       a__   		=	TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

       if model_mapping is not None:
              a__   		=	{config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
       if tf_model_mapping is not None:
              a__   		=	{
                  config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
              }

       def 							_A	(		self			:Tuple       ,							lowerCAmelCase__			:Tuple       ,							lowerCAmelCase__			:Union[str, Any]       ,							lowerCAmelCase__			:Dict				)				->  Any:


          '''simple docstring'''




          snake_case_				:					Optional[Any]   =       ZeroShotClassificationPipeline(
              model=lowerCAmelCase__       ,							tokenizer=lowerCAmelCase__       ,							candidate_labels=["polics", "health"]				)
          return classifier, ["Who are you voting for in 2020?", "My stomach hurts."]

       def 							_A	(		self			:List[str]       ,							lowerCAmelCase__			:Optional[int]       ,							lowerCAmelCase__			:Any				)				->  Optional[Any]:


          '''simple docstring'''




          snake_case_				:					Tuple   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels="politics"				)
          self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

          # No kwarg
          snake_case_				:					List[Any]   =       classifier("Who are you voting for in 2020?"       ,							["politics"]				)
          self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

          snake_case_				:					Dict   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels=["politics"]				)
          self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

          snake_case_				:					int   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels="politics, public health"				)
          self.assertEqual(
              lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}				)
          self.assertAlmostEqual(sum(nested_simplify(outputs["scores"]				)				)       ,							1.0				)

          snake_case_				:					Optional[int]   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health"]				)
          self.assertEqual(
              lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}				)
          self.assertAlmostEqual(sum(nested_simplify(outputs["scores"]				)				)       ,							1.0				)

          snake_case_				:					str   =       classifier(
              "Who are you voting for in 2020?"       ,							candidate_labels="politics"       ,							hypothesis_template="This text is about {}"				)
          self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

          # https://github.com/huggingface/transformers/issues/13846
          snake_case_				:					Dict   =       classifier(["I am happy"]       ,							["positive", "negative"]				)
          self.assertEqual(
              lowerCAmelCase__       ,							[
                  {"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}
                  for i in range(1				)
              ]       ,							)
          snake_case_				:					Tuple   =       classifier(["I am happy", "I am sad"]       ,							["positive", "negative"]				)
          self.assertEqual(
              lowerCAmelCase__       ,							[
                  {"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}
                  for i in range(2				)
              ]       ,							)

          with self.assertRaises(lowerCAmelCase__				):
             classifier(""       ,							candidate_labels="politics"				)

          with self.assertRaises(lowerCAmelCase__				):
             classifier(lowerCAmelCase__       ,							candidate_labels="politics"				)

          with self.assertRaises(lowerCAmelCase__				):
             classifier("Who are you voting for in 2020?"       ,							candidate_labels=""				)

          with self.assertRaises(lowerCAmelCase__				):
             classifier("Who are you voting for in 2020?"       ,							candidate_labels=lowerCAmelCase__				)

          with self.assertRaises(lowerCAmelCase__				):
             classifier(
                 "Who are you voting for in 2020?"       ,							candidate_labels="politics"       ,							hypothesis_template="Not formatting template"       ,							)

          with self.assertRaises(lowerCAmelCase__				):
             classifier(
                 "Who are you voting for in 2020?"       ,							candidate_labels="politics"       ,							hypothesis_template=lowerCAmelCase__       ,							)

          self.run_entailment_id(lowerCAmelCase__				)

       def 							_A	(		self			:List[Any]       ,							lowerCAmelCase__			:Pipeline				)				->  Union[str, Any]:


          '''simple docstring'''




          snake_case_				:					int   =       zero_shot_classifier.model.config
          snake_case_				:					Optional[int]   =       config.labelaid
          snake_case_				:					Tuple   =       zero_shot_classifier.entailment_id

          snake_case_				:					Optional[Any]   =       {"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2}
          self.assertEqual(zero_shot_classifier.entailment_id       ,							-1				)

          snake_case_				:					Tuple   =       {"entailment": 0, "neutral": 1, "contradiction": 2}
          self.assertEqual(zero_shot_classifier.entailment_id       ,							0				)

          snake_case_				:					str   =       {"ENTAIL": 0, "NON-ENTAIL": 1}
          self.assertEqual(zero_shot_classifier.entailment_id       ,							0				)

          snake_case_				:					str   =       {"ENTAIL": 2, "NEUTRAL": 1, "CONTR": 0}
          self.assertEqual(zero_shot_classifier.entailment_id       ,							2				)

          snake_case_				:					List[str]   =       original_labelaid
          self.assertEqual(lowerCAmelCase__       ,							zero_shot_classifier.entailment_id				)

       @require_torch
       def 							_A	(		self			:Tuple				)				->  Any:


          '''simple docstring'''




          snake_case_				:					List[Any]   =       pipeline(
              "zero-shot-classification"       ,							model="sshleifer/tiny-distilbert-base-cased-distilled-squad"       ,							framework="pt"       ,							)
          # There was a regression in 4.10 for this
          # Adding a test so we don't make the mistake again.
          # https://github.com/huggingface/transformers/issues/13381#issuecomment-912343499
          zero_shot_classifier(
              "Who are you voting for in 2020?" * 100       ,							candidate_labels=["politics", "public health", "science"]				)

       @require_torch
       def 							_A	(		self			:Optional[Any]				)				->  Tuple:


          '''simple docstring'''




          snake_case_				:					Union[str, Any]   =       pipeline(
              "zero-shot-classification"       ,							model="sshleifer/tiny-distilbert-base-cased-distilled-squad"       ,							framework="pt"       ,							)
          snake_case_				:					int   =       zero_shot_classifier(
              "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

          self.assertEqual(
              nested_simplify(lowerCAmelCase__				)       ,							{
                  "sequence": "Who are you voting for in 2020?",
                  "labels": ["science", "public health", "politics"],
                  "scores": [0.3_3_3, 0.3_3_3, 0.3_3_3],
              }       ,							)

       @require_tf
       def 							_A	(		self			:Union[str, Any]				)				->  Dict:


          '''simple docstring'''




          snake_case_				:					List[str]   =       pipeline(
              "zero-shot-classification"       ,							model="sshleifer/tiny-distilbert-base-cased-distilled-squad"       ,							framework="tf"       ,							)
          snake_case_				:					Optional[int]   =       zero_shot_classifier(
              "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

          self.assertEqual(
              nested_simplify(lowerCAmelCase__				)       ,							{
                  "sequence": "Who are you voting for in 2020?",
                  "labels": ["science", "public health", "politics"],
                  "scores": [0.3_3_3, 0.3_3_3, 0.3_3_3],
              }       ,							)

       @slow
       @require_torch
       def 							_A	(		self			:Union[str, Any]				)				->  int:


          '''simple docstring'''




          snake_case_				:					int   =       pipeline("zero-shot-classification"       ,							model="roberta-large-mnli"       ,							framework="pt"				)
          snake_case_				:					str   =       zero_shot_classifier(
              "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

          self.assertEqual(
              nested_simplify(lowerCAmelCase__				)       ,							{
                  "sequence": "Who are you voting for in 2020?",
                  "labels": ["politics", "public health", "science"],
                  "scores": [0.9_7_6, 0.0_1_5, 0.0_0_9],
              }       ,							)
          snake_case_				:					Optional[int]   =       zero_shot_classifier(
              "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
              " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
              " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
              " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
              " machine translation tasks show these models to be superior in quality while being more parallelizable"
              " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
              " English-to-German translation task, improving over the existing best results, including ensembles by"
              " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
              " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
              " fraction of the training costs of the best models from the literature. We show that the Transformer"
              " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
              " large and limited training data."       ,							candidate_labels=["machine learning", "statistics", "translation", "vision"]       ,							multi_label=lowerCAmelCase__       ,							)
          self.assertEqual(
              nested_simplify(lowerCAmelCase__				)       ,							{
                  "sequence": (
                      "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
                      " networks in an encoder-decoder configuration. The best performing models also connect the"
                      " encoder and decoder through an attention mechanism. We propose a new simple network"
                      " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
                      " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
                      " superior in quality while being more parallelizable and requiring significantly less time to"
                      " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
                      " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
                      " English-to-French translation task, our model establishes a new single-model state-of-the-art"
                      " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
                      " costs of the best models from the literature. We show that the Transformer generalizes well to"
                      " other tasks by applying it successfully to English constituency parsing both with large and"
                      " limited training data."
                  ),
                  "labels": ["translation", "machine learning", "vision", "statistics"],
                  "scores": [0.8_1_7, 0.7_1_3, 0.0_1_8, 0.0_1_8],
              }       ,							)






       @slow
       @require_tf
       def 							_A	(		self			:List[str]				)				->  str:


          '''simple docstring'''




          snake_case_				:					int   =       pipeline("zero-shot-classification"       ,							model="roberta-large-mnli"       ,							framework="tf"				)
          snake_case_				:					Optional[Any]   =       zero_shot_classifier(
              "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

          self.assertEqual(
              nested_simplify(lowerCAmelCase__				)       ,							{
                  "sequence": "Who are you voting for in 2020?",
                  "labels": ["politics", "public health", "science"],
                  "scores": [0.9_7_6, 0.0_1_5, 0.0_0_9],
              }       ,							)
          snake_case_				:					Tuple   =       zero_shot_classifier(
              "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
              " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
              " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
              " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
              " machine translation tasks show these models to be superior in quality while being more parallelizable"
              " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
              " English-to-German translation task, improving over the existing best results, including ensembles by"
              " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
              " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
              " fraction of the training costs of the best models from the literature. We show that the Transformer"
              " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
              " large and limited training data."       ,							candidate_labels=["machine learning", "statistics", "translation", "vision"]       ,							multi_label=lowerCAmelCase__       ,							)
          self.assertEqual(
              nested_simplify(lowerCAmelCase__				)       ,							{
                  "sequence": (
                      "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
                      " networks in an encoder-decoder configuration. The best performing models also connect the"
                      " encoder and decoder through an attention mechanism. We propose a new simple network"
                      " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
                      " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
                      " superior in quality while being more parallelizable and requiring significantly less time to"
                      " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
                      " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
                      " English-to-French translation task, our model establishes a new single-model state-of-the-art"
                      " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
                      " costs of the best models from the literature. We show that the Transformer generalizes well to"
                      " other tasks by applying it successfully to English constituency parsing both with large and"
                      " limited training data."
                  ),
                  "labels": ["translation", "machine learning", "vision", "statistics"],
                  "scores": [0.8_1_7, 0.7_1_3, 0.0_1_8, 0.0_1_8],
              }       ,							)












'''simple docstring'''




import sacrebleu as scb
from packaging import version
from sacrebleu import TER

import datasets


__lowerCamelCase   :					str   							=					'''\
@inproceedings{snover-etal-2006-study,
    title = "A Study of Translation Edit Rate with Targeted Human Annotation",
    author = "Snover, Matthew  and
      Dorr, Bonnie  and
      Schwartz, Rich  and
      Micciulla, Linnea  and
      Makhoul, John",
    booktitle = "Proceedings of the 7th Conference of the Association for Machine Translation in the Americas: Technical Papers",
    month = aug # " 8-12",
    year = "2006",
    address = "Cambridge, Massachusetts, USA",
    publisher = "Association for Machine Translation in the Americas",
    url = "https://aclanthology.org/2006.amta-papers.25",
    pages = "223--231",
}
@inproceedings{post-2018-call,
    title = "A Call for Clarity in Reporting {BLEU} Scores",
    author = "Post, Matt",
    booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers",
    month = oct,
    year = "2018",
    address = "Belgium, Brussels",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/W18-6319",
    pages = "186--191",
}
'''

__lowerCamelCase   :					Dict   							=					'''\
TER (Translation Edit Rate, also called Translation Error Rate) is a metric to quantify the edit operations that a
hypothesis requires to match a reference translation. We use the implementation that is already present in sacrebleu
(https://github.com/mjpost/sacreBLEU#ter), which in turn is inspired by the TERCOM implementation, which can be found
here: https://github.com/jhclark/tercom.

The implementation here is slightly different from sacrebleu in terms of the required input format. The length of
the references and hypotheses lists need to be the same, so you may need to transpose your references compared to
sacrebleu\'s required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534

See the README.md file at https://github.com/mjpost/sacreBLEU#ter for more information.
'''

__lowerCamelCase   :					int   							=					'''
Produces TER scores alongside the number of edits and reference length.

Args:
    predictions (list of str): The system stream (a sequence of segments).
    references (list of list of str): A list of one or more reference streams (each a sequence of segments).
    normalized (boolean): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`.
    ignore_punct (boolean): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`.
    support_zh_ja_chars (boolean): If `True`, tokenization/normalization supports processing of Chinese characters,
                                    as well as Japanese Kanji, Hiragana, Katakana, and Phonetic Extensions of Katakana.
                                    Only applies if `normalized = True`. Defaults to `False`.
    case_sensitive (boolean): If `False`, makes all predictions and references lowercase to ignore differences in case. Defaults to `False`.

Returns:
    \'score\' (float): TER score (num_edits / sum_ref_lengths * 100)
    \'num_edits\' (int): The cumulative number of edits
    \'ref_length\' (float): The cumulative average reference length

Examples:
    Example 1:
        >>> predictions = ["does this sentence match??",
        ...                     "what about this sentence?",
        ...                     "What did the TER metric user say to the developer?"]
        >>> references = [["does this sentence match", "does this sentence match!?!"],
        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"],
        ...             ["Your jokes are...", "...TERrible"]]
        >>> ter = datasets.load_metric("ter")
        >>> results = ter.compute(predictions=predictions,
        ...                         references=references,
        ...                         case_sensitive=True)
        >>> print(results)
        {\'score\': 150.0, \'num_edits\': 15, \'ref_length\': 10.0}

    Example 2:
        >>> predictions = ["does this sentence match??",
        ...                     "what about this sentence?"]
        >>> references = [["does this sentence match", "does this sentence match!?!"],
        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]]
        >>> ter = datasets.load_metric("ter")
        >>> results = ter.compute(predictions=predictions,
        ...                         references=references,
        ...                         case_sensitive=True)
        >>> print(results)
        {\'score\': 62.5, \'num_edits\': 5, \'ref_length\': 8.0}

    Example 3:
        >>> predictions = ["does this sentence match??",
        ...                     "what about this sentence?"]
        >>> references = [["does this sentence match", "does this sentence match!?!"],
        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]]
        >>> ter = datasets.load_metric("ter")
        >>> results = ter.compute(predictions=predictions,
        ...                         references=references,
        ...                         normalized=True,
        ...                         case_sensitive=True)
        >>> print(results)
        {\'score\': 57.14285714285714, \'num_edits\': 6, \'ref_length\': 10.5}

    Example 4:
        >>> predictions = ["does this sentence match??",
        ...                     "what about this sentence?"]
        >>> references = [["does this sentence match", "does this sentence match!?!"],
        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]]
        >>> ter = datasets.load_metric("ter")
        >>> results = ter.compute(predictions=predictions,
        ...                         references=references,
        ...                         ignore_punct=True,
        ...                         case_sensitive=False)
        >>> print(results)
        {\'score\': 0.0, \'num_edits\': 0, \'ref_length\': 8.0}

    Example 5:
        >>> predictions = ["does this sentence match??",
        ...                    "what about this sentence?",
        ...                    "What did the TER metric user say to the developer?"]
        >>> references = [["does this sentence match", "does this sentence match!?!"],
        ...             ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"],
        ...             ["Your jokes are...", "...TERrible"]]
        >>> ter = datasets.load_metric("ter")
        >>> results = ter.compute(predictions=predictions,
        ...                         references=references,
        ...                         ignore_punct=True,
        ...                         case_sensitive=False)
        >>> print(results)
        {\'score\': 100.0, \'num_edits\': 10, \'ref_length\': 10.0}
'''







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						,					_KWARGS_DESCRIPTION							)
class    A_	(datasets.Metric							):
							"""simple docstring"""




							def 							_A	(		self			:Optional[Any]				)				->  Optional[int]:


										'''simple docstring'''




										if version.parse(scb.__version__				) < version.parse("1.4.12"				):
													raise ImportWarning(
													    "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n"
													    "You can install it with `pip install \"sacrebleu>=1.4.12\"`."				)
										return datasets.MetricInfo(
										    description=_DESCRIPTION       ,							citation=_CITATION       ,							homepage="http://www.cs.umd.edu/~snover/tercom/"       ,							inputs_description=_KWARGS_DESCRIPTION       ,							features=datasets.Features(
										        {
										            "predictions": datasets.Value("string"       ,							id="sequence"				),
										            "references": datasets.Sequence(datasets.Value("string"       ,							id="sequence"				)       ,							id="references"				),
										        }				)       ,							codebase_urls=["https://github.com/mjpost/sacreBLEU#ter"]       ,							reference_urls=[
										        "https://github.com/jhclark/tercom",
										    ]       ,							)






							def 							_A	(		self			:Any       ,							lowerCAmelCase__			:Any       ,							lowerCAmelCase__			:str       ,							lowerCAmelCase__			:bool = False       ,							lowerCAmelCase__			:bool = False       ,							lowerCAmelCase__			:bool = False       ,							lowerCAmelCase__			:bool = False       ,							)				->  Optional[Any]:


										'''simple docstring'''




										snake_case_				:					List[str]   =       len(references[0]				)
										if any(len(lowerCAmelCase__				) != references_per_prediction for refs in references				):
													raise ValueError("Sacrebleu requires the same number of references for each prediction"				)
										snake_case_				:					List[str]   =       [[refs[i] for refs in references] for i in range(lowerCAmelCase__				)]

										snake_case_				:					List[str]   =       TER(
										    normalized=lowerCAmelCase__       ,							no_punct=lowerCAmelCase__       ,							asian_support=lowerCAmelCase__       ,							case_sensitive=lowerCAmelCase__       ,							)
										snake_case_				:					Any   =       sb_ter.corpus_score(lowerCAmelCase__       ,							lowerCAmelCase__				)

										return {"score": output.score, "num_edits": output.num_edits, "ref_length": output.ref_length}




