import random
import unittest

import numpy as np

import transformers
from transformers import is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax


if is_flax_available():
      import os

      import jax.numpy as jnp
      from jax import jit

      from transformers import AutoTokenizer, FlaxAutoModelForCausalLM
      from transformers.modeling_flax_pytorch_utils import load_flax_weights_in_pytorch_model

      SCREAMING_SNAKE_CASE__		:					Optional[Any]						=					"0.12"  # assumed parallelism: 8


if is_torch_available():
      import torch







def     _a				(      lowercase__  :			Optional[Any]     ,  lowercase__  :			Tuple     ,  lowercase__  :			List[str]=None ):







     '''simple docstring'''


     if rng is None:
          SCREAMING_SNAKE_CASE__				:				Optional[int]			=		random.Random()

     SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		1
     for dim in shape:
          total_dims *= dim

     SCREAMING_SNAKE_CASE__				:				Tuple			=		[]
     for _ in range(lowercase__ ):
          values.append(rng.randint(0     ,  vocab_size - 1 ) )

     SCREAMING_SNAKE_CASE__				:				Optional[int]			=		np.array(lowercase__     ,  dtype=jnp.intaa ).reshape(lowercase__ )

     return output







def     _a				(      lowercase__  :			Optional[Any]     ,  lowercase__  :			int=None ):







     '''simple docstring'''


     SCREAMING_SNAKE_CASE__				:				str			=		ids_tensor(lowercase__     ,  vocab_size=2     ,  rng=lowercase__ )
     # make sure that at least one token is attended to for each batch
     SCREAMING_SNAKE_CASE__				:				List[Any]			=		1
     return attn_mask

@require_flax
class       snake_case					:
  lowercase_			=     None
  lowercase_			=     ()




  def 	__lowercase(      self     :	str				)->       Tuple:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__				:				int			=		self.model_tester.prepare_config_and_inputs_for_common()

       # cut to half length & take max batch_size 3
       SCREAMING_SNAKE_CASE__				:				str			=		2
       SCREAMING_SNAKE_CASE__				:				int			=		inputs['input_ids'].shape[-1] // 2
       SCREAMING_SNAKE_CASE__				:				Any			=		inputs['input_ids'][:max_batch_size, :sequence_length]

       SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		jnp.ones_like(a_				)
       SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		attention_mask[:max_batch_size, :sequence_length]

       # generate max 5 tokens
       SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		input_ids.shape[-1] + 5
       if config.eos_token_id is not None and config.pad_token_id is None:
            # hack to allow generate for models such as GPT2 as is done in `generate()`
            SCREAMING_SNAKE_CASE__				:				Tuple			=		config.eos_token_id
       return config, input_ids, attention_mask, max_length




  @is_pt_flax_cross_test
  def 	__lowercase(      self     :	List[str]				)->       Optional[Any]:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		self._get_input_ids_and_config()
       SCREAMING_SNAKE_CASE__				:				int			=		False
       SCREAMING_SNAKE_CASE__				:				str			=		max_length
       SCREAMING_SNAKE_CASE__				:				Dict			=		0

       for model_class in self.all_generative_model_classes:
            SCREAMING_SNAKE_CASE__				:				Any			=		model_class(a_				)

            SCREAMING_SNAKE_CASE__				:				int			=		model_class.__name__[4:]  # Skip the "Flax" at the beginning
            SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		getattr(a_     , a_				)
            SCREAMING_SNAKE_CASE__				:				List[str]			=		pt_model_class(a_				).eval()
            SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		load_flax_weights_in_pytorch_model(a_     , flax_model.params				)

            SCREAMING_SNAKE_CASE__				:				Dict			=		flax_model.generate(a_				).sequences
            SCREAMING_SNAKE_CASE__				:				str			=		pt_model.generate(torch.tensor(a_     , dtype=torch.long				)				)

            if flax_generation_outputs.shape[-1] > pt_generation_outputs.shape[-1]:
                 SCREAMING_SNAKE_CASE__				:				List[Any]			=		flax_generation_outputs[:, : pt_generation_outputs.shape[-1]]

            self.assertListEqual(pt_generation_outputs.numpy().tolist()     , flax_generation_outputs.tolist()				)




  def 	__lowercase(      self     :	Optional[Any]				)->       int:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		self._get_input_ids_and_config()
       SCREAMING_SNAKE_CASE__				:				List[str]			=		False
       SCREAMING_SNAKE_CASE__				:				str			=		max_length

       for model_class in self.all_generative_model_classes:
            SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		model_class(a_				)

            SCREAMING_SNAKE_CASE__				:				str			=		model.generate(a_				).sequences
            self.assertEqual(generation_outputs.shape[-1]     , a_				)

            SCREAMING_SNAKE_CASE__				:				List[str]			=		jit(model.generate				)
            SCREAMING_SNAKE_CASE__				:				Dict			=		jit_generate(a_				).sequences

            self.assertListEqual(generation_outputs.tolist()     , jit_generation_outputs.tolist()				)




  def 	__lowercase(      self     :	Tuple				)->       Union[str, Any]:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__				:				int			=		self._get_input_ids_and_config()
       SCREAMING_SNAKE_CASE__				:				List[str]			=		True
       SCREAMING_SNAKE_CASE__				:				str			=		max_length

       for model_class in self.all_generative_model_classes:
            SCREAMING_SNAKE_CASE__				:				List[str]			=		model_class(a_				)

            SCREAMING_SNAKE_CASE__				:				Optional[int]			=		model.generate(a_				).sequences
            self.assertEqual(generation_outputs.shape[-1]     , a_				)

            SCREAMING_SNAKE_CASE__				:				Any			=		jit(model.generate				)
            SCREAMING_SNAKE_CASE__				:				str			=		jit_generate(a_				).sequences

            self.assertListEqual(generation_outputs.tolist()     , jit_generation_outputs.tolist()				)




  def 	__lowercase(      self     :	int				)->       Union[str, Any]:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		self._get_input_ids_and_config()
       SCREAMING_SNAKE_CASE__				:				int			=		False
       SCREAMING_SNAKE_CASE__				:				str			=		max_length
       SCREAMING_SNAKE_CASE__				:				Any			=		2

       for model_class in self.all_generative_model_classes:
            SCREAMING_SNAKE_CASE__				:				Dict			=		model_class(a_				)

            SCREAMING_SNAKE_CASE__				:				str			=		model.generate(a_				).sequences
            self.assertEqual(generation_outputs.shape[-1]     , a_				)

            SCREAMING_SNAKE_CASE__				:				Optional[int]			=		jit(model.generate				)
            SCREAMING_SNAKE_CASE__				:				Tuple			=		jit_generate(a_				).sequences

            self.assertListEqual(generation_outputs.tolist()     , jit_generation_outputs.tolist()				)




  def 	__lowercase(      self     :	Union[str, Any]				)->       Tuple:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__				:				int			=		self._get_input_ids_and_config()
       SCREAMING_SNAKE_CASE__				:				Any			=		False
       SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		max_length
       SCREAMING_SNAKE_CASE__				:				Tuple			=		2
       SCREAMING_SNAKE_CASE__				:				Tuple			=		2

       for model_class in self.all_generative_model_classes:
            SCREAMING_SNAKE_CASE__				:				List[Any]			=		model_class(a_				)

            SCREAMING_SNAKE_CASE__				:				Dict			=		model.generate(a_				).sequences
            self.assertEqual(generation_outputs.shape[0]     , input_ids.shape[0] * config.num_return_sequences				)




  def 	__lowercase(      self     :	Any				)->       Any:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__				:				Dict			=		self._get_input_ids_and_config()
       SCREAMING_SNAKE_CASE__				:				str			=		True
       SCREAMING_SNAKE_CASE__				:				int			=		max_length
       SCREAMING_SNAKE_CASE__				:				List[str]			=		0.8
       SCREAMING_SNAKE_CASE__				:				Tuple			=		10
       SCREAMING_SNAKE_CASE__				:				str			=		0.3
       SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		1
       SCREAMING_SNAKE_CASE__				:				Any			=		8
       SCREAMING_SNAKE_CASE__				:				Dict			=		9

       for model_class in self.all_generative_model_classes:
            SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		model_class(a_				)

            SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		model.generate(a_				).sequences
            self.assertEqual(generation_outputs.shape[-1]     , a_				)

            SCREAMING_SNAKE_CASE__				:				Any			=		jit(model.generate				)
            SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		jit_generate(a_				).sequences

            self.assertListEqual(generation_outputs.tolist()     , jit_generation_outputs.tolist()				)




  def 	__lowercase(      self     :	Dict				)->       Any:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__				:				Dict			=		self._get_input_ids_and_config()
       SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		max_length
       SCREAMING_SNAKE_CASE__				:				Tuple			=		1
       SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		8
       SCREAMING_SNAKE_CASE__				:				str			=		9

       for model_class in self.all_generative_model_classes:
            SCREAMING_SNAKE_CASE__				:				Dict			=		model_class(a_				)

            SCREAMING_SNAKE_CASE__				:				Dict			=		model.generate(a_				).sequences
            self.assertEqual(generation_outputs.shape[-1]     , a_				)

            SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		jit(model.generate				)
            SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		jit_generate(a_				).sequences

            self.assertListEqual(generation_outputs.tolist()     , jit_generation_outputs.tolist()				)




  def 	__lowercase(      self     :	Optional[Any]				)->       List[Any]:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__				:				Optional[int]			=		self._get_input_ids_and_config()
       SCREAMING_SNAKE_CASE__				:				List[Any]			=		max_length
       SCREAMING_SNAKE_CASE__				:				Tuple			=		2
       SCREAMING_SNAKE_CASE__				:				List[Any]			=		1
       SCREAMING_SNAKE_CASE__				:				Any			=		8
       SCREAMING_SNAKE_CASE__				:				Optional[int]			=		9

       for model_class in self.all_generative_model_classes:
            SCREAMING_SNAKE_CASE__				:				List[Any]			=		model_class(a_				)

            SCREAMING_SNAKE_CASE__				:				int			=		model.generate(a_				).sequences
            self.assertEqual(generation_outputs.shape[-1]     , a_				)

            SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		jit(model.generate				)
            SCREAMING_SNAKE_CASE__				:				int			=		jit_generate(a_				).sequences

            self.assertListEqual(generation_outputs.tolist()     , jit_generation_outputs.tolist()				)




  def 	__lowercase(      self     :	str				)->       str:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__				:				Tuple			=		self._get_input_ids_and_config()

       # pad attention mask on the left
       SCREAMING_SNAKE_CASE__				:				List[str]			=		attention_mask.at[(0, 0)].set(0				)

       SCREAMING_SNAKE_CASE__				:				Any			=		False
       SCREAMING_SNAKE_CASE__				:				Optional[int]			=		max_length

       for model_class in self.all_generative_model_classes:
            SCREAMING_SNAKE_CASE__				:				str			=		model_class(a_				)

            SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		model.generate(a_     , attention_mask=a_				).sequences
            self.assertEqual(generation_outputs.shape[-1]     , a_				)

            SCREAMING_SNAKE_CASE__				:				Optional[int]			=		jit(model.generate				)
            SCREAMING_SNAKE_CASE__				:				Tuple			=		jit_generate(a_     , attention_mask=a_				).sequences

            self.assertListEqual(generation_outputs.tolist()     , jit_generation_outputs.tolist()				)




  def 	__lowercase(      self     :	List[str]				)->       List[str]:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__				:				List[Any]			=		self._get_input_ids_and_config()

       # pad attention mask on the left
       SCREAMING_SNAKE_CASE__				:				List[Any]			=		attention_mask.at[(0, 0)].set(0				)

       SCREAMING_SNAKE_CASE__				:				List[str]			=		True
       SCREAMING_SNAKE_CASE__				:				Dict			=		max_length

       for model_class in self.all_generative_model_classes:
            SCREAMING_SNAKE_CASE__				:				Dict			=		model_class(a_				)

            SCREAMING_SNAKE_CASE__				:				int			=		model.generate(a_     , attention_mask=a_				).sequences
            self.assertEqual(generation_outputs.shape[-1]     , a_				)

            SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		jit(model.generate				)
            SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		jit_generate(a_     , attention_mask=a_				).sequences

            self.assertListEqual(generation_outputs.tolist()     , jit_generation_outputs.tolist()				)




  def 	__lowercase(      self     :	Any				)->       List[Any]:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__				:				str			=		self._get_input_ids_and_config()

       # pad attention mask on the left
       SCREAMING_SNAKE_CASE__				:				List[str]			=		attention_mask.at[(0, 0)].set(0				)

       SCREAMING_SNAKE_CASE__				:				Any			=		2
       SCREAMING_SNAKE_CASE__				:				List[Any]			=		max_length

       for model_class in self.all_generative_model_classes:
            SCREAMING_SNAKE_CASE__				:				Tuple			=		model_class(a_				)

            SCREAMING_SNAKE_CASE__				:				Any			=		model.generate(a_     , attention_mask=a_				).sequences
            self.assertEqual(generation_outputs.shape[-1]     , a_				)

            SCREAMING_SNAKE_CASE__				:				List[Any]			=		jit(model.generate				)
            SCREAMING_SNAKE_CASE__				:				Dict			=		jit_generate(a_     , attention_mask=a_				).sequences

            self.assertListEqual(generation_outputs.tolist()     , jit_generation_outputs.tolist()				)




@require_flax
class       snake_case					(      unittest.TestCase						):




  def 	__lowercase(      self     :	Optional[Any]				)->       int:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__				:				List[str]			=		AutoTokenizer.from_pretrained('hf-internal-testing/tiny-bert'				)
       SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		FlaxAutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-bert-flax-only'				)

       SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		'Hello world'
       SCREAMING_SNAKE_CASE__				:				List[str]			=		tokenizer(a_     , return_tensors='np'				).input_ids

       # typos are quickly detected (the correct argument is `do_sample`)
       with self.assertRaisesRegex(a_     , 'do_samples'				):
            model.generate(a_     , do_samples=a_				)

       # arbitrary arguments that will not be used anywhere are also not accepted
       with self.assertRaisesRegex(a_     , 'foo'				):
            SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		{'foo': 'bar'}
            model.generate(a_     , **a_				)




from dataclasses import dataclass
from typing import Tuple

import numpy as np
import torch

@dataclass
class       snake_case					:
  lowercase_			=     42  # [batch_size x 3]
  lowercase_			=     42  # [batch_size x 3]
  lowercase_			=     42  # [batch_size x 3]
  lowercase_			=     42  # [batch_size x 3]
  lowercase_			=     42
  lowercase_			=     42
  lowercase_			=     42
  lowercase_			=     42
  lowercase_			=     42




  def 	__lowercase(      self     :	List[Any]				)->       Union[str, Any]:




       """simple docstring"""
       assert self.x.shape[0] == self.y.shape[0] == self.z.shape[0] == self.origin.shape[0]
       assert self.x.shape[1] == self.y.shape[1] == self.z.shape[1] == self.origin.shape[1] == 3
       assert len(self.x.shape				) == len(self.y.shape				) == len(self.z.shape				) == len(self.origin.shape				) == 2




  def 	__lowercase(      self     :	Dict				)->       Tuple:




       """simple docstring"""
       return torch.from_numpy(np.array([self.width, self.height]     , dtype=np.floataa				)				)




  def 	__lowercase(      self     :	Dict				)->       Union[str, Any]:




       """simple docstring"""
       return torch.from_numpy(np.array([self.x_fov, self.y_fov]     , dtype=np.floataa				)				)




  def 	__lowercase(      self     :	Tuple				)->       torch.Tensor:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__				:				str			=		torch.arange(self.height * self.width				)
       SCREAMING_SNAKE_CASE__				:				List[Any]			=		torch.stack(
           [
               pixel_indices % self.width,
               torch.div(a_     , self.width     , rounding_mode='trunc'				),
           ]     , axis=1     , )
       return coords




  @property
  def 	__lowercase(      self     :	Any				)->       List[Any]:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__			,							*SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		self.shape
       SCREAMING_SNAKE_CASE__				:				Tuple			=		int(np.prod(a_				)				)

       SCREAMING_SNAKE_CASE__				:				List[str]			=		self.get_image_coords()
       SCREAMING_SNAKE_CASE__				:				Dict			=		torch.broadcast_to(coords.unsqueeze(0				)     , [batch_size * inner_batch_size, *coords.shape]				)
       SCREAMING_SNAKE_CASE__				:				Any			=		self.get_camera_rays(a_				)

       SCREAMING_SNAKE_CASE__				:				Tuple			=		rays.view(a_     , inner_batch_size * self.height * self.width     , 2     , 3				)

       return rays




  def 	__lowercase(      self     :	Optional[Any]     , a_     :	torch.Tensor				)->       torch.Tensor:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__			,							*SCREAMING_SNAKE_CASE__			,							SCREAMING_SNAKE_CASE__				:				Dict			=		coords.shape
       assert n_coords == 2
       assert batch_size == self.origin.shape[0]

       SCREAMING_SNAKE_CASE__				:				str			=		coords.view(a_     , -1     , 2				)

       SCREAMING_SNAKE_CASE__				:				List[Any]			=		self.resolution()
       SCREAMING_SNAKE_CASE__				:				str			=		self.fov()

       SCREAMING_SNAKE_CASE__				:				Any			=		(flat.float() / (res - 1)) * 2 - 1
       SCREAMING_SNAKE_CASE__				:				Any			=		fracs * torch.tan(fov / 2				)

       SCREAMING_SNAKE_CASE__				:				List[str]			=		fracs.view(a_     , -1     , 2				)
       SCREAMING_SNAKE_CASE__				:				str			=		(
           self.z.view(a_     , 1     , 3				)
           + self.x.view(a_     , 1     , 3				) * fracs[:, :, :1]
           + self.y.view(a_     , 1     , 3				) * fracs[:, :, 1:]
       )
       SCREAMING_SNAKE_CASE__				:				Tuple			=		directions / directions.norm(dim=-1     , keepdim=a_				)
       SCREAMING_SNAKE_CASE__				:				Any			=		torch.stack(
           [
               torch.broadcast_to(self.origin.view(a_     , 1     , 3				)     , [batch_size, directions.shape[1], 3]				),
               directions,
           ]     , dim=2     , )
       return rays.view(a_     , *a_     , 2     , 3				)




  def 	__lowercase(      self     :	Optional[int]     , a_     :	int     , a_     :	int				)->       "DifferentiableProjectiveCamera":




       """simple docstring"""
       assert width * self.height == height * self.width, "The aspect ratio should not change."
       return DifferentiableProjectiveCamera(
           origin=self.origin     , x=self.x     , y=self.y     , z=self.z     , width=a_     , height=a_     , x_fov=self.x_fov     , y_fov=self.y_fov     , )







def     _a				(      lowercase__  :			int ):







     '''simple docstring'''


     SCREAMING_SNAKE_CASE__				:				Any			=		[]
     SCREAMING_SNAKE_CASE__				:				List[Any]			=		[]
     SCREAMING_SNAKE_CASE__				:				Optional[int]			=		[]
     SCREAMING_SNAKE_CASE__				:				str			=		[]
     for theta in np.linspace(0     ,  2 * np.pi     ,  num=20 ):
          SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		np.array([np.sin(lowercase__ ), np.cos(lowercase__ ), -0.5] )
          z /= np.sqrt(np.sum(z**2 ) )
          SCREAMING_SNAKE_CASE__				:				Tuple			=		-z * 4
          SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		np.array([np.cos(lowercase__ ), -np.sin(lowercase__ ), 0.0] )
          SCREAMING_SNAKE_CASE__				:				Optional[int]			=		np.cross(lowercase__     ,  lowercase__ )
          origins.append(lowercase__ )
          xs.append(lowercase__ )
          ys.append(lowercase__ )
          zs.append(lowercase__ )
     return DifferentiableProjectiveCamera(
         origin=torch.from_numpy(np.stack(lowercase__     ,  axis=0 ) ).float()     ,  x=torch.from_numpy(np.stack(lowercase__     ,  axis=0 ) ).float()     ,  y=torch.from_numpy(np.stack(lowercase__     ,  axis=0 ) ).float()     ,  z=torch.from_numpy(np.stack(lowercase__     ,  axis=0 ) ).float()     ,  width=lowercase__     ,  height=lowercase__     ,  x_fov=0.7     ,  y_fov=0.7     ,  shape=(1, len(lowercase__ ))     ,  )



