





'''simple docstring'''




import unittest

from transformers import MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING, AutoTokenizer, is_vision_available
from transformers.pipelines import pipeline
from transformers.pipelines.document_question_answering import apply_tesseract
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_detectrona,
    require_pytesseract,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
    from PIL import Image

    from transformers.image_utils import load_image
else:






    class UpperCAmelCase							:



        @staticmethod
        def 					__lowerCamelCase						(  *__A     ,			**__A					):
         pass







    def _lowerCAmelCase     (      _lowerCAmelCase    )->     Optional[Any]:
     return None


# This is a pinned image from a specific revision of a document question answering space, hosted by HuggingFace,
# so we can expect it to be available.
_A:					Any							=				(
    """https://huggingface.co/spaces/impira/docquery/resolve/2f6c96314dc84dfda62d40de9da55f2f5165d403/invoice.png"""
)






@is_pipeline_test
@require_torch
@require_vision
class UpperCAmelCase							(		unittest.TestCase							):
    _A :			List[str]						   =       MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING



    @require_pytesseract
    @require_vision
    def 					__lowerCamelCase						(  self     ,			__A     ,			__A     ,			__A					):
     __UpperCAmelCase					=   pipeline(
         'document-question-answering'     ,			model=__A     ,			tokenizer=__A     ,			image_processor=__A					)

     __UpperCAmelCase					=   INVOICE_URL
     __UpperCAmelCase					=   list(zip(*apply_tesseract(load_image(__A					)     ,			__A     ,			''					)					)					)
     __UpperCAmelCase					=   'What is the placebo?'
     __UpperCAmelCase					=   [
         {
             'image': load_image(__A					),
             'question': question,
         },
         {
             'image': image,
             'question': question,
         },
         {
             'image': image,
             'question': question,
             'word_boxes': word_boxes,
         },
     ]
     return dqa_pipeline, examples



    def 					__lowerCamelCase						(  self     ,			__A     ,			__A					):
     __UpperCAmelCase					=   dqa_pipeline(__A     ,			top_k=2					)
     self.assertEqual(
         __A     ,			[
             [
                 {'score': ANY(__A					), 'answer': ANY(__A					), 'start': ANY(__A					), 'end': ANY(__A					)},
                 {'score': ANY(__A					), 'answer': ANY(__A					), 'start': ANY(__A					), 'end': ANY(__A					)},
             ]
         ]
         * 3     ,			)



    @require_torch
    @require_detectrona
    @require_pytesseract
    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   pipeline('document-question-answering'     ,			model='hf-internal-testing/tiny-random-layoutlmv2'					)
     __UpperCAmelCase					=   INVOICE_URL
     __UpperCAmelCase					=   'How many cats are there?'

     __UpperCAmelCase					=   [
         {'score': 0.0_0_0_1, 'answer': 'oy 2312/2019', 'start': 38, 'end': 39},
         {'score': 0.0_0_0_1, 'answer': 'oy 2312/2019 DUE', 'start': 38, 'end': 40},
     ]
     __UpperCAmelCase					=   dqa_pipeline(image=__A     ,			question=__A     ,			top_k=2					)
     self.assertEqual(nested_simplify(__A     ,			decimals=4					)     ,			__A					)

     __UpperCAmelCase					=   dqa_pipeline({'image': image, 'question': question}     ,			top_k=2					)
     self.assertEqual(nested_simplify(__A     ,			decimals=4					)     ,			__A					)

     # This image does not detect ANY text in it, meaning layoutlmv2 should fail.
     # Empty answer probably
     __UpperCAmelCase					=   './tests/fixtures/tests_samples/COCO/000000039769.png'
     __UpperCAmelCase					=   dqa_pipeline(image=__A     ,			question=__A     ,			top_k=2					)
     self.assertEqual(__A     ,			[]					)

     # We can optionnally pass directly the words and bounding boxes
     __UpperCAmelCase					=   './tests/fixtures/tests_samples/COCO/000000039769.png'
     __UpperCAmelCase					=   []
     __UpperCAmelCase					=   []
     __UpperCAmelCase					=   dqa_pipeline(image=__A     ,			question=__A     ,			words=__A     ,			boxes=__A     ,			top_k=2					)
     self.assertEqual(__A     ,			[]					)



    @slow
    @require_torch
    @require_detectrona
    @require_pytesseract
    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   pipeline(
         'document-question-answering'     ,			model='tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa'     ,			revision='9977165'     ,			)
     __UpperCAmelCase					=   INVOICE_URL
     __UpperCAmelCase					=   'What is the invoice number?'

     __UpperCAmelCase					=   dqa_pipeline(image=__A     ,			question=__A     ,			top_k=2					)
     self.assertEqual(
         nested_simplify(__A     ,			decimals=4					)     ,			[
             {'score': 0.9_9_4_4, 'answer': 'us-001', 'start': 16, 'end': 16},
             {'score': 0.0_0_0_9, 'answer': 'us-001', 'start': 16, 'end': 16},
         ]     ,			)

     __UpperCAmelCase					=   dqa_pipeline({'image': image, 'question': question}     ,			top_k=2					)
     self.assertEqual(
         nested_simplify(__A     ,			decimals=4					)     ,			[
             {'score': 0.9_9_4_4, 'answer': 'us-001', 'start': 16, 'end': 16},
             {'score': 0.0_0_0_9, 'answer': 'us-001', 'start': 16, 'end': 16},
         ]     ,			)

     __UpperCAmelCase					=   dqa_pipeline(
         [{'image': image, 'question': question}, {'image': image, 'question': question}]     ,			top_k=2					)
     self.assertEqual(
         nested_simplify(__A     ,			decimals=4					)     ,			[
             [
                 {'score': 0.9_9_4_4, 'answer': 'us-001', 'start': 16, 'end': 16},
                 {'score': 0.0_0_0_9, 'answer': 'us-001', 'start': 16, 'end': 16},
             ],
         ]
         * 2     ,			)



    @slow
    @require_torch
    @require_detectrona
    @require_pytesseract
    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   pipeline(
         'document-question-answering'     ,			model='tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa'     ,			revision='9977165'     ,			max_seq_len=50     ,			)
     __UpperCAmelCase					=   INVOICE_URL
     __UpperCAmelCase					=   'What is the invoice number?'

     __UpperCAmelCase					=   dqa_pipeline(image=__A     ,			question=__A     ,			top_k=2					)
     self.assertEqual(
         nested_simplify(__A     ,			decimals=4					)     ,			[
             {'score': 0.9_9_7_4, 'answer': '1110212019', 'start': 23, 'end': 23},
             {'score': 0.9_9_4_8, 'answer': 'us-001', 'start': 16, 'end': 16},
         ]     ,			)

     __UpperCAmelCase					=   dqa_pipeline({'image': image, 'question': question}     ,			top_k=2					)
     self.assertEqual(
         nested_simplify(__A     ,			decimals=4					)     ,			[
             {'score': 0.9_9_7_4, 'answer': '1110212019', 'start': 23, 'end': 23},
             {'score': 0.9_9_4_8, 'answer': 'us-001', 'start': 16, 'end': 16},
         ]     ,			)

     __UpperCAmelCase					=   dqa_pipeline(
         [{'image': image, 'question': question}, {'image': image, 'question': question}]     ,			top_k=2					)
     self.assertEqual(
         nested_simplify(__A     ,			decimals=4					)     ,			[
             [
                 {'score': 0.9_9_7_4, 'answer': '1110212019', 'start': 23, 'end': 23},
                 {'score': 0.9_9_4_8, 'answer': 'us-001', 'start': 16, 'end': 16},
             ]
         ]
         * 2     ,			)



    @slow
    @require_torch
    @require_pytesseract
    @require_vision
    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   AutoTokenizer.from_pretrained(
         'impira/layoutlm-document-qa'     ,			revision='3dc6de3'     ,			add_prefix_space=__A					)
     __UpperCAmelCase					=   pipeline(
         'document-question-answering'     ,			model='impira/layoutlm-document-qa'     ,			tokenizer=__A     ,			revision='3dc6de3'     ,			)
     __UpperCAmelCase					=   INVOICE_URL
     __UpperCAmelCase					=   'What is the invoice number?'

     __UpperCAmelCase					=   dqa_pipeline(image=__A     ,			question=__A     ,			top_k=2					)
     self.assertEqual(
         nested_simplify(__A     ,			decimals=4					)     ,			[
             {'score': 0.4_2_5_1, 'answer': 'us-001', 'start': 16, 'end': 16},
             {'score': 0.0_8_1_9, 'answer': '1110212019', 'start': 23, 'end': 23},
         ]     ,			)

     __UpperCAmelCase					=   dqa_pipeline({'image': image, 'question': question}     ,			top_k=2					)
     self.assertEqual(
         nested_simplify(__A     ,			decimals=4					)     ,			[
             {'score': 0.4_2_5_1, 'answer': 'us-001', 'start': 16, 'end': 16},
             {'score': 0.0_8_1_9, 'answer': '1110212019', 'start': 23, 'end': 23},
         ]     ,			)

     __UpperCAmelCase					=   dqa_pipeline(
         [{'image': image, 'question': question}, {'image': image, 'question': question}]     ,			top_k=2					)
     self.assertEqual(
         nested_simplify(__A     ,			decimals=4					)     ,			[
             [
                 {'score': 0.4_2_5_1, 'answer': 'us-001', 'start': 16, 'end': 16},
                 {'score': 0.0_8_1_9, 'answer': '1110212019', 'start': 23, 'end': 23},
             ]
         ]
         * 2     ,			)

     __UpperCAmelCase					=   list(zip(*apply_tesseract(load_image(__A					)     ,			__A     ,			''					)					)					)

     # This model should also work if `image` is set to None
     __UpperCAmelCase					=   dqa_pipeline({'image': None, 'word_boxes': word_boxes, 'question': question}     ,			top_k=2					)
     self.assertEqual(
         nested_simplify(__A     ,			decimals=4					)     ,			[
             {'score': 0.4_2_5_1, 'answer': 'us-001', 'start': 16, 'end': 16},
             {'score': 0.0_8_1_9, 'answer': '1110212019', 'start': 23, 'end': 23},
         ]     ,			)



    @slow
    @require_torch
    @require_pytesseract
    @require_vision
    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   AutoTokenizer.from_pretrained(
         'impira/layoutlm-document-qa'     ,			revision='3dc6de3'     ,			add_prefix_space=__A					)
     __UpperCAmelCase					=   pipeline(
         'document-question-answering'     ,			model='impira/layoutlm-document-qa'     ,			tokenizer=__A     ,			revision='3dc6de3'     ,			max_seq_len=50     ,			)
     __UpperCAmelCase					=   INVOICE_URL
     __UpperCAmelCase					=   'What is the invoice number?'

     __UpperCAmelCase					=   dqa_pipeline(image=__A     ,			question=__A     ,			top_k=2					)
     self.assertEqual(
         nested_simplify(__A     ,			decimals=4					)     ,			[
             {'score': 0.9_9_9_9, 'answer': 'us-001', 'start': 16, 'end': 16},
             {'score': 0.9_9_9_8, 'answer': 'us-001', 'start': 16, 'end': 16},
         ]     ,			)

     __UpperCAmelCase					=   dqa_pipeline(
         [{'image': image, 'question': question}, {'image': image, 'question': question}]     ,			top_k=2					)
     self.assertEqual(
         nested_simplify(__A     ,			decimals=4					)     ,			[
             [
                 {'score': 0.9_9_9_9, 'answer': 'us-001', 'start': 16, 'end': 16},
                 {'score': 0.9_9_9_8, 'answer': 'us-001', 'start': 16, 'end': 16},
             ]
         ]
         * 2     ,			)

     __UpperCAmelCase					=   list(zip(*apply_tesseract(load_image(__A					)     ,			__A     ,			''					)					)					)

     # This model should also work if `image` is set to None
     __UpperCAmelCase					=   dqa_pipeline({'image': None, 'word_boxes': word_boxes, 'question': question}     ,			top_k=2					)
     self.assertEqual(
         nested_simplify(__A     ,			decimals=4					)     ,			[
             {'score': 0.9_9_9_9, 'answer': 'us-001', 'start': 16, 'end': 16},
             {'score': 0.9_9_9_8, 'answer': 'us-001', 'start': 16, 'end': 16},
         ]     ,			)



    @slow
    @require_torch
    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   pipeline(
         'document-question-answering'     ,			model='naver-clova-ix/donut-base-finetuned-docvqa'     ,			tokenizer=AutoTokenizer.from_pretrained('naver-clova-ix/donut-base-finetuned-docvqa'					)     ,			feature_extractor='naver-clova-ix/donut-base-finetuned-docvqa'     ,			)

     __UpperCAmelCase					=   INVOICE_URL
     __UpperCAmelCase					=   'What is the invoice number?'
     __UpperCAmelCase					=   dqa_pipeline(image=__A     ,			question=__A     ,			top_k=2					)
     self.assertEqual(nested_simplify(__A     ,			decimals=4					)     ,			[{'answer': 'us-001'}]					)




    @require_tf
    @unittest.skip('Document question answering not implemented in TF'					)
    def 					__lowerCamelCase						(  self					):
     pass












'''simple docstring'''




import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
    import sqlitea

    import sqlalchemy






class UpperCAmelCase							(		UpperCAmelCase_							):



    def __init__(  self     ,			__A     ,			__A     ,			__A = None     ,			__A = None     ,			__A = False     ,			**__A     ,			):
     super().__init__(features=__A     ,			cache_dir=__A     ,			keep_in_memory=__A     ,			**__A					)
     __UpperCAmelCase					=   Sql(
         cache_dir=__A     ,			features=__A     ,			sql=__A     ,			con=__A     ,			**__A     ,			)




    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   None
     __UpperCAmelCase					=   None
     __UpperCAmelCase					=   None
     __UpperCAmelCase					=   None

     self.builder.download_and_prepare(
         download_config=__A     ,			download_mode=__A     ,			verification_mode=__A     ,			base_path=__A     ,			)

     # Build dataset for splits
     __UpperCAmelCase					=   self.builder.as_dataset(
         split='train'     ,			verification_mode=__A     ,			in_memory=self.keep_in_memory					)
     return dataset






class UpperCAmelCase							:



    def __init__(  self     ,			__A     ,			__A     ,			__A     ,			__A = None     ,			__A = None     ,			**__A     ,			):
     if num_proc is not None and num_proc <= 0:
      raise ValueError(f'num_proc {num_proc} must be an integer > 0.'					)

     __UpperCAmelCase					=   dataset
     __UpperCAmelCase					=   name
     __UpperCAmelCase					=   con
     __UpperCAmelCase					=   batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
     __UpperCAmelCase					=   num_proc
     __UpperCAmelCase					=   to_sql_kwargs



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   self.to_sql_kwargs.pop('sql'     ,			__A					)
     __UpperCAmelCase					=   self.to_sql_kwargs.pop('con'     ,			__A					)
     __UpperCAmelCase					=   self.to_sql_kwargs.pop('index'     ,			__A					)

     __UpperCAmelCase					=   self._write(index=__A     ,			**self.to_sql_kwargs					)
     return written



    def 					__lowerCamelCase						(  self     ,			__A					):
     __UpperCAmelCase					,    __UpperCAmelCase					,    __UpperCAmelCase					=   args
     __UpperCAmelCase					=   {**to_sql_kwargs, 'if_exists': 'append'} if offset > 0 else to_sql_kwargs
     __UpperCAmelCase					=   query_table(
         table=self.dataset.data     ,			key=slice(__A     ,			offset + self.batch_size					)     ,			indices=self.dataset._indices     ,			)
     __UpperCAmelCase					=   batch.to_pandas()
     __UpperCAmelCase					=   df.to_sql(self.name     ,			self.con     ,			index=__A     ,			**__A					)
     return num_rows or len(__A					)




    def 					__lowerCamelCase						(  self     ,			__A     ,			**__A					):
     __UpperCAmelCase					=   0

     if self.num_proc is None or self.num_proc == 1:
      for offset in logging.tqdm(
          range(0     ,			len(self.dataset					)     ,			self.batch_size					)     ,			unit='ba'     ,			disable=not logging.is_progress_bar_enabled()     ,			desc='Creating SQL from Arrow format'     ,			):
       written += self._batch_sql((offset, index, to_sql_kwargs)					)
     else:
      __UpperCAmelCase					,    __UpperCAmelCase					=   len(self.dataset					), self.batch_size
      with multiprocessing.Pool(self.num_proc					) as pool:
       for num_rows in logging.tqdm(
           pool.imap(
               self._batch_sql     ,			[(offset, index, to_sql_kwargs) for offset in range(0     ,			__A     ,			__A					)]     ,			)     ,			total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size     ,			unit='ba'     ,			disable=not logging.is_progress_bar_enabled()     ,			desc='Creating SQL from Arrow format'     ,			):
        written += num_rows

     return written





