CoolFace
Apppublic

Aluode/PerceptionLabPortable

sourceHugging Faceupdated 9mo agoView on Hugging Face
0likes
rootpython_embedLibsite-packagestransformersmodelsfnet__pycache__modeling_fnet.cpython-310.pyc
modeling_fnet.cpython-310.pyc365 linesDownload Raw Back to __pycache__
1o

2/�Yi���@s�dZddlZddlmZddlmZddlmZmZddl	Z	ddl	m3Z4ddlmZm
Z
mZdd	lmZmZe�r?dd5lmZddlmZddlmZdd
lmZmZmZmZmZmZmZm Z m!Z!ddl"m#Z#ddl$m%Z%ddlm&Z&ddl'm(Z(e&�)e*�Z+dd�Z,dd�Z-dd�Z.Gdd�de6j/�Z0Gdd�de7j/�Z1Gdd�de8j/�Z2Gdd �d e9j/�Z3Gd!d"�d"e10j/�Z4Gd#d$�d$e11j/�Z5Gd%d&�d&e�Z6Gd'd(�d(e12j/�Z7Gd)d*�d*e13j/�Z8Gd+d,�d,e14j/�Z9Gd-d.�d.e15j/�Z:Gd/d0�d0e16j/�Z;Gd1d2�d2e17j/�Z<Gd3d4�d4e18j/�Z=eGd5d6�d6e#��Z>eed7d8�Gd9d:�d:e���Z?eGd;d<�d<e>��Z@ed=d8�Gd>d?�d?e>��ZAeGd@dA�dAe>��ZBedBd8�GdCdD�dDe>��ZCedEd8�GdFdG�dGe>��ZDeGdHdI�dIe>��ZEeGdJdK�dKe>��ZFeGdLdM�dMe>��ZGgdN�ZHdS)OzPyTorch FNet model.�N)�	dataclass)�partial)�Optional�Union)�nn)�BCEWithLogitsLoss�CrossEntropyLoss�MSELoss�)�auto_docstring�is_scipy_available)�linalg)�ACT2FN)�GradientCheckpointingLayer)	�BaseModelOutput�BaseModelOutputWithPooling�MaskedLMOutput�ModelOutput�MultipleChoiceModelOutput�NextSentencePredictorOutput�QuestionAnsweringModelOutput�SequenceClassifierOutput�TokenClassifierOutput)�PreTrainedModel)�apply_chunking_to_forward)�logging�)�19FNetConfigcCs:|jd}|d|�d|�f}|�tj�}t�d|||�S)z4Applies 2D matrix multiplication to 3D input arrays.rNzbij,jk,ni->bnk)�shape�type�torch�	complex64Zeinsum)�x�matrix_dim_one�matrix_dim_two�20seq_length�r&��E:\DocsHouse\542 percep lab latest\PerceptionLab\PerceptionLab_Portable\python_embed\Lib\site-packages\transformers/models/fnet/modeling_fnet.py�_two_dim_matmul7s21r(cCst|||�S�N)r()r"r#r$r&r&r'�two_dim_matmul@�r*cCs4|}tt|j�dd��D]22}tjj||d�}q
|S)z�23    Applies n-dimensional Fast Fourier Transform (FFT) to input array.24 25    Args:26        x: Input n-dimensional array.27 28    Returns:29        n-dimensional Fourier transform of input n-dimensional array.30    rN)�axis)�reversed�range�ndimr �fft)r"�outr,r&r&r'�fftnEs31r2cs*eZdZdZ�fdd�Zddd�Z�ZS)�FNetEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.cs�t���tj|j|j|jd�|_t�|j|j�|_	t�|j32|j�|_tj|j|j
d�|_t�|j|j�|_t�|j�|_|jdt�|j��d�dd�|jdtj|j��tjd�dd�dS)	N)�padding_idx��eps�position_ids)r�����F)�33persistent�token_type_ids��dtype)�super�__init__r�	Embedding�34vocab_size�hidden_sizeZpad_token_id�word_embeddings�max_position_embeddings�position_embeddingsZtype_vocab_size�token_type_embeddings�	LayerNorm�layer_norm_eps�Linear�35projection�Dropout�hidden_dropout_prob�dropout�register_bufferr Zarange�expand�zerosr7�size�long��self�config��	__class__r&r'r>Xs36�37�zFNetEmbeddings.__init__NcCs�|dur	|��}n|��dd�}|d}|dur$|jdd�d|�f}|durNt|d�rC|jdd�d|�f}|�|d|�}|}ntj|tj|jjd�}|durW|�	|�}|�38|�}	||	}39|�|�}|40|7}41|�|42�}43|�
|44�}45|�|46�}47|48S)Nr8rr:r�r<�device)rPr7�hasattrr:rNr rOrQrXrBrErDrFrIrL)rS�	input_idsr:r7�
inputs_embeds�input_shaper%�buffered_token_type_ids� buffered_token_type_ids_expandedrE�49embeddingsrDr&r&r'�forwardns,5051525354555657zFNetEmbeddings.forward)NNNN)�__name__�58__module__�__qualname__�__doc__r>r`�
__classcell__r&r&rUr'r3Usr3c�,eZdZ�fdd�Zdd�Zdd�Z�ZS)�FNetBasicFourierTransformcst���|�|�dSr))r=r>�_init_fourier_transformrRrUr&r'r>��59z"FNetBasicFourierTransform.__init__cCs�|jsttjjdd�|_dS|jdkrLt�rB|�dtj	t60�|j�tj
d��|�dtj	t61�|j�tj
d��tt|j|jd�|_dSt�d�t|_dSt|_dS)	N)r���dim��dft_mat_hiddenr;�dft_mat_seq)r#r$zpSciPy is needed for DFT matrix calculation and is not found. Using TPU optimized fast fourier transform instead.)�use_tpu_fourier_optimizationsrr r0r2�fourier_transformrCrrMZtensorr
ZdftrAr!�tpu_short_seq_lengthr*rornr�warningrRr&r&r'rh�s$62��63��6465z1FNetBasicFourierTransform._init_fourier_transformcCs|�|�j}|fSr))rq�real)rS�
hidden_states�outputsr&r&r'r`�sz!FNetBasicFourierTransform.forward)rarbrcr>rhr`rer&r&rUr'rg�srgc�$eZdZ�fdd�Zdd�Z�ZS)�FNetBasicOutputcs"t���tj|j|jd�|_dS�Nr5)r=r>rrFrArGrRrUr&r'r>�s66zFNetBasicOutput.__init__cCs|�||�}|Sr))rF�rSru�input_tensorr&r&r'r`�szFNetBasicOutput.forward�rarbrcr>r`rer&r&rUr'rx��rxcrw)�FNetFourierTransformcs"t���t|�|_t|�|_dSr))r=r>rgrSrx�outputrRrUr&r'r>�s6768zFNetFourierTransform.__init__cCs$|�|�}|�|d|�}|f}|S�Nr)rSr)rSruZself_outputs�fourier_outputrvr&r&r'r`�s69zFNetFourierTransform.forwardr|r&r&rUr'r~��r~c�2eZdZ�fdd�Zdejdejfdd�Z�ZS)�FNetIntermediatecsDt���t�|j|j�|_t|jt	�rt70|j|_dS|j|_dSr))r=r>rrHrA�intermediate_size�dense�71isinstance�72hidden_act�strr�intermediate_act_fnrRrUr&r'r>�s7374zFNetIntermediate.__init__ru�returncC�|�|�}|�|�}|Sr))r�r��rSrur&r&r'r`��7576zFNetIntermediate.forward�rarbrcr>r �Tensorr`rer&r&rUr'r��sr�cs8eZdZ�fdd�Zdejdejdejfdd�Z�ZS)�77FNetOutputcsBt���t�|j|j�|_tj|j|jd�|_t�	|j78�|_dSry)r=r>rrHr�rAr�rFrGrJrKrLrRrUr&r'r>�s79zFNetOutput.__init__rur{r�cCs&|�|�}|�|�}|�||�}|Sr))r�rLrFrzr&r&r'r`�s8081zFNetOutput.forwardr�r&r&rUr'r��s$r�crf)�	FNetLayercs:t���|j|_d|_t|�|_t|�|_t|�|_	dS�Nr)82r=r>�chunk_size_feed_forward�seq_len_dimr~�fourierr��intermediater�rrRrUr&r'r>�s838485zFNetLayer.__init__cCs0|�|�}|d}t|j|j|j|�}|f}|Sr�)r�r�feed_forward_chunkr�r�)rSruZself_fourier_outputsr��layer_outputrvr&r&r'r`�s86�zFNetLayer.forwardcCs|�|�}|�||�}|Sr))r�r)rSr�Zintermediate_outputr�r&r&r'r�s87zFNetLayer.feed_forward_chunk)rarbrcr>r`r�rer&r&rUr'r��sr�cs&eZdZ�fdd�Zddd�Z�ZS)�FNetEncodercs:t����|_t��fdd�t�j�D��|_d|_dS)Ncsg|]}t���qSr&)r�)�.0�_�rTr&r'�88<listcomp>
sz(FNetEncoder.__init__.<locals>.<listcomp>F)	r=r>rTrZ89ModuleListr.Znum_hidden_layers�layerZgradient_checkpointingrRrUr�r'r>90s91 92zFNetEncoder.__init__FTcCsr|rdnd}t|j�D]\}}|r||f}||�}|d}q|r&||f}|s3tdd�||fD��St||d�S)Nr&rcss�|]	}|dur|VqdSr)r&)r��vr&r&r'�	<genexpr>s�z&FNetEncoder.forward.<locals>.<genexpr>)�last_hidden_stateru)�	enumerater��tupler)rSru�output_hidden_states�return_dictZall_hidden_states�iZlayer_moduleZ
layer_outputsr&r&r'r`s939495zFNetEncoder.forward)FTr|r&r&rUr'r�	sr�cr�)�96FNetPoolercs*t���t�|j|j�|_t��|_dSr))r=r>rrHrAr�ZTanh�97activationrRrUr&r'r>&s98zFNetPooler.__init__rur�cCs(|dd�df}|�|�}|�|�}|Sr�)r�r�)rSruZfirst_token_tensor�
pooled_outputr&r&r'r`+s99100zFNetPooler.forwardr�r&r&rUr'r�%sr�cr�)�FNetPredictionHeadTransformcsVt���t�|j|j�|_t|jt�rt	|j|_101n|j|_102tj|j|jd�|_dSry)
r=r>rrHrAr�r�r�r�r�transform_act_fnrFrGrRrUr&r'r>6s103z$FNetPredictionHeadTransform.__init__rur�cCs"|�|�}|�|�}|�|�}|Sr))r�r�rFr�r&r&r'r`?s104105106z#FNetPredictionHeadTransform.forwardr�r&r&rUr'r�5s	r�cs.eZdZ�fdd�Zdd�Zd	dd�Z�ZS)107�FNetLMPredictionHeadcsHt���t|�|_t�|j|j�|_t�	t108�|j��|_|j|j_dSr))
r=r>r��	transformrrHrAr@�decoder�	Parameterr rO�biasrRrUr&r'r>Gs109110111zFNetLMPredictionHead.__init__cCr�r))r�r�r�r&r&r'r`Rr�zFNetLMPredictionHead.forwardr�NcCs,|jjjjdkr|j|j_dS|jj|_dS)N�meta)r�r�rXr�rSr&r&r'�_tie_weightsWsz!FNetLMPredictionHead._tie_weights)r�N)rarbrcr>r`r�rer&r&rUr'r�Fsr�crw)�FNetOnlyMLMHeadcst���t|�|_dSr))r=r>r��predictionsrRrUr&r'r>arizFNetOnlyMLMHead.__init__cC�|�|�}|Sr))r�)rS�sequence_output�prediction_scoresr&r&r'r`e�112zFNetOnlyMLMHead.forwardr|r&r&rUr'r�`r}r�crw)�FNetOnlyNSPHeadcst���t�|jd�|_dS�Nrj)r=r>rrHrA�seq_relationshiprRrUr&r'r>ls113zFNetOnlyNSPHead.__init__cCr�r))r�)rSr��seq_relationship_scorer&r&r'r`pr�zFNetOnlyNSPHead.forwardr|r&r&rUr'r�kr}r�crw)�FNetPreTrainingHeadscs(t���t|�|_t�|jd�|_dSr�)r=r>r�r�rrHrAr�rRrUr&r'r>ws114115zFNetPreTrainingHeads.__init__cCs|�|�}|�|�}||fSr))r�r�)rSr�r�r�r�r&r&r'r`|s116117zFNetPreTrainingHeads.forwardr|r&r&rUr'r�vr�r�c@s&eZdZUeed<dZdZdd�ZdS)�FNetPreTrainedModelrT�fnetTcCs�t|tj�r |jjjd|jjd�|jdur|jj�	�dSdSt|tj118�rC|jjjd|jjd�|jdurA|jj|j�	�dSdSt|tj�rX|jj�	�|jj�
d�dSdS)zInitialize the weightsg)�meanZstdNg�?)r�rrH�weight�dataZnormal_rTZinitializer_ranger�Zzero_r?r4rFZfill_)rS�moduler&r&r'�
_init_weights�s119�120��z!FNetPreTrainedModel._init_weightsN)rarbrcr�__annotations__Zbase_model_prefixZsupports_gradient_checkpointingr�r&r&r&r'r��s121122r�z0123    Output type of [`FNetForPreTraining`].124    )Zcustom_introc@s^eZdZUdZdZeejed<dZ	eejed<dZ125eejed<dZeeejed<dS)�FNetForPreTrainingOutputa�126    loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):127        Total loss as the sum of the masked language modeling loss and the next sequence prediction128        (classification) loss.129    prediction_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):130        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).131    seq_relationship_logits (`torch.FloatTensor` of shape `(batch_size, 2)`):132        Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation133        before SoftMax).134    N�loss�prediction_logits�seq_relationship_logitsru)
rarbrcrdr�rr �FloatTensorr�r�r�rur�r&r&r&r'r��s135r�cs�eZdZdZd�fdd�	Zdd�Zdd�Ze												dd136ee	j137dee	j138dee	j139d
ee	jdeedeede
eeffdd��Z�ZS)�	FNetModelz�140 141    The model can behave as an encoder, following the architecture described in [FNet: Mixing Tokens with Fourier142    Transforms](https://huggingface.co/papers/2105.03824) by James Lee-Thorp, Joshua Ainslie, Ilya Eckstein, Santiago Ontanon.143 144    TcsDt��|�||_t|�|_t|�|_|rt|�nd|_|�	�dS)zv145        add_pooling_layer (bool, *optional*, defaults to `True`):146            Whether to add a pooling layer147        N)148r=r>rTr3r_r��encoderr��pooler�	post_init)rSrTZadd_pooling_layerrUr&r'r>�s149150zFNetModel.__init__cCs|jjSr)�r_rBr�r&r&r'�get_input_embeddings�szFNetModel.get_input_embeddingscCs||j_dSr)r�)rS�valuer&r&r'�set_input_embeddings�r+zFNetModel.set_input_embeddingsNrZr:r7r[r�r�r�cCsv|dur|n|jj}|dur|n|jj}|dur |dur td��|dur-|��}|\}}	n|dur>|��dd�}|\}}	ntd��|jjrT|	dkrT|jj|	krTtd��|dur[|jn|j}151|dur�t|j	d�r}|j	j152dd�d|	�f}|�||	�}|}n	tj
|tj|153d�}|j	||||d�}
|j|
||d	�}|d154}|jdur�|�|�nd}|s�||f|dd�St|||jd�S)
NzDYou cannot specify both input_ids and inputs_embeds at the same timer8z5You have to specify either input_ids or inputs_embedsrmz�The `tpu_short_seq_length` in FNetConfig should be set equal to the sequence length being passed to the model when using TPU optimizations.r:rW)rZr7r:r[)r�r�rr)r��
pooler_outputru)rTr��use_return_dict�155ValueErrorrPrprrrXrYr_r:rNr rOrQr�r�rru)rSrZr:r7r[r�r�r\Z156batch_sizer%rXr]r^Zembedding_outputZencoder_outputsr�r�r&r&r'r`�s\�157158�����zFNetModel.forward)T)NNNNNN)rarbrcrdr>r�r�rrr Z159LongTensorr��boolrr�rr`rer&r&rUr'r��s6�������160�r�z�161    FNet Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next162    sentence prediction (classification)` head.163    cs�eZdZddgZ�fdd�Zdd�Zdd�Ze																dd164ee	j165dee	j166dee	j167d
ee	j168dee	j169dee	j170deedeedee
effdd��Z�ZS)�FNetForPreTraining�cls.predictions.decoder.bias�cls.predictions.decoder.weightc�,t��|�t|�|_t|�|_|��dSr))r=r>r�r�r��clsr�rRrUr&r'r>!�171172zFNetForPreTraining.__init__cC�173|jjjSr)�r�r�r�r�r&r&r'�get_output_embeddings*�174z(FNetForPreTraining.get_output_embeddingscC�||jj_|j|jj_dSr)�r�r�r�r��rSZnew_embeddingsr&r&r'�set_output_embeddings-�175z(FNetForPreTraining.set_output_embeddingsNrZr:r7r[�labels�next_sentence_labelr�r�r�c	Cs�|dur|n|jj}|j||||||d�}	|	dd�\}176}|�|177|�\}}
d}|durP|durPt�}||�d|jj�|�d��}||
�dd�|�d��}||}|sg||
f|	dd�}|dure|f|S|St|||
|	jd�S)aH178        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):179            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,180            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the181            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`182        next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):183            Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair184            (see `input_ids` docstring) Indices should be in `[0, 1]`:185 186            - 0 indicates sequence B is a continuation of sequence A,187            - 1 indicates sequence B is a random sequence.188 189        Example:190 191        ```python192        >>> from transformers import AutoTokenizer, FNetForPreTraining193        >>> import torch194 195        >>> tokenizer = AutoTokenizer.from_pretrained("google/fnet-base")196        >>> model = FNetForPreTraining.from_pretrained("google/fnet-base")197        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")198        >>> outputs = model(**inputs)199        >>> prediction_logits = outputs.prediction_logits200        >>> seq_relationship_logits = outputs.seq_relationship_logits201        ```N�r:r7r[r�r�rjr8)r�r�r�ru)	rTr�r�r�r�viewr@r�ru)rSrZr:r7r[r�r�r�r�rvr�r�r�r��202total_loss�loss_fct�masked_lm_loss�next_sentence_lossrr&r&r'r`1s4%�	�zFNetForPreTraining.forward�NNNNNNNN)rarbrc�_tied_weights_keysr>r�r�rrr r�r�rr�r�r`rer&r&rUr'r�sB	��������	�203 204�r�cs�eZdZddgZ�fdd�Zdd�Zdd�Ze														dd205ee	j206dee	j207dee	j208d
ee	j209dee	j210deedeedee
effdd��Z�ZS)�FNetForMaskedLMr�r�cr�r))r=r>r�r�r�r�r�rRrUr&r'r>{r�zFNetForMaskedLM.__init__cCr�r)r�r�r&r&r'r��r�z%FNetForMaskedLM.get_output_embeddingscCr�r)r�r�r&r&r'r��r�z%FNetForMaskedLM.set_output_embeddingsNrZr:r7r[r�r�r�r�cCs�|dur|n|jj}|j||||||d�}|d}	|�|	�}211d}|dur5t�}||212�d|jj�|�d��}|sK|213f|dd�}
|durI|f|
S|
St||214|jd�S)a�215        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):216            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,217            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the218            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.219        Nr�rr8rj�r��logitsru)	rTr�r�r�rr�r@rru)rSrZr:r7r[r�r�r�rvr�r�r�r�rr&r&r'r`�s&�	220zFNetForMaskedLM.forward�NNNNNNN)rarbrcr�r>r�r�rrr r�r�rr�rr`rer&r&rUr'r�ws<	��������221	�r�zT222    FNet Model with a `next sentence prediction (classification)` head on top.223    c�eZdZ�fdd�Ze							ddeejdeejdeejdeejdeejd	eed224eede	e225effdd
��Z�Z
S)�FNetForNextSentencePredictioncr�r))r=r>r�r�r�r�r�rRrUr&r'r>�r�z&FNetForNextSentencePrediction.__init__NrZr:r7r[r�r�r�r�cKs�d|vrt�dt�|�d�}|dur|n|jj}|j||||||d�}	|	d}226|�|227�}d}|durBt�}
|
|�	dd�|�	d��}|sX|f|	dd�}|durV|f|S|St228|||	jd�S)	a�229        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):230            Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair231            (see `input_ids` docstring). Indices should be in `[0, 1]`:232 233            - 0 indicates sequence B is a continuation of sequence A,234            - 1 indicates sequence B is a random sequence.235 236        Example:237 238        ```python239        >>> from transformers import AutoTokenizer, FNetForNextSentencePrediction240        >>> import torch241 242        >>> tokenizer = AutoTokenizer.from_pretrained("google/fnet-base")243        >>> model = FNetForNextSentencePrediction.from_pretrained("google/fnet-base")244        >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."245        >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."246        >>> encoding = tokenizer(prompt, next_sentence, return_tensors="pt")247        >>> outputs = model(**encoding, labels=torch.LongTensor([1]))248        >>> logits = outputs.logits249        >>> assert logits[0, 0] < logits[0, 1]  # next sentence was random250        ```r�zoThe `next_sentence_label` argument is deprecated and will be removed in a future version, use `labels` instead.Nr�rr8rjr�)�warnings�warn�
FutureWarning�poprTr�r�r�rr�rru)rSrZr:r7r[r�r�r��kwargsrvr�Zseq_relationship_scoresr�r�rr&r&r'r`�s:$�251�	252�z%FNetForNextSentencePrediction.forwardr�)rarbrcr>rrr r�r�rr�rr`rer&r&rUr'r��s6	��������253 254�r�z�255    FNet Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled256    output) e.g. for GLUE tasks.257    cr�)�FNetForSequenceClassificationc�Jt��|�|j|_t|�|_t�|j�|_t�	|j258|j�|_|��dSr)�
r=r>�259num_labelsr�r�rrJrKrLrHrA�260classifierr�rRrUr&r'r>s261z&FNetForSequenceClassification.__init__NrZr:r7r[r�r�r�r�cCsh|dur|n|jj}|j||||||d�}|d}	|�|	�}	|�|	�}262d}|dur�|jjdurS|jdkr9d|j_n|jdkrO|jtj	ksJ|jtj263krOd|j_nd|j_|jjdkrqt�}|jdkrk||264��|���}n+||265|�}n%|jjdkr�t
�}||266�d|j�|�d��}n|jjdkr�t�}||267|�}|s�|268f|dd�}
|dur�|f|
S|
St||269|jd	�S)270a�271        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):272            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,273            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If274            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).275        Nr�rZ276regressionZsingle_label_classificationZmulti_label_classificationr8rjr�)rTr�r�rLrZproblem_typerr<r rQ�intr	�squeezerr�rrru)rSrZr:r7r[r�r�r�rvr�r�r�r�rr&r&r'r`"sF�	277278279280"281282283z%FNetForSequenceClassification.forwardr�)rarbrcr>rrr r�r�rr�rr`rer&r&rUr'r�s6��������284	�r�cr�)�FNetForMultipleChoicecs@t��|�t|�|_t�|j�|_t�|j	d�|_285|��dSr�)r=r>r�r�rrJrKrLrHrArr�rRrUr&r'r>as286287zFNetForMultipleChoice.__init__NrZr:r7r[r�r�r�r�cCsF|dur|n|jj}|dur|jdn|jd}|dur%|�d|�d��nd}|dur4|�d|�d��nd}|durC|�d|�d��nd}|durV|�d|�d�|�d��nd}|j||||||d�}	|	d}288|�|289�}290|�|291�}|�d|�}d}
|dur�t�}|||�}
|s�|f|	dd�}|
dur�|
f|S|St	|
||	j292d�S)a[293        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):294            Indices of input sequence tokens in the vocabulary.295 296            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and297            [`PreTrainedTokenizer.__call__`] for details.298 299            [What are input IDs?](../glossary#input-ids)300        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):301            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,302            1]`:303 304            - 0 corresponds to a *sentence A* token,305            - 1 corresponds to a *sentence B* token.306 307            [What are token type IDs?](../glossary#token-type-ids)308        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):309            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,310            config.max_position_embeddings - 1]`.311 312            [What are position IDs?](../glossary#position-ids)313        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):314            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This315            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the316            model's internal embedding lookup matrix.317        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):318            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,319            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See320            `input_ids` above)321        Nrr8�����r�rjr�)rTr�rr�rPr�rLrrrru)rSrZr:r7r[r�r�r�Znum_choicesrvr�r�Zreshaped_logitsr�r�rr&r&r'r`ks:)���	322323324zFNetForMultipleChoice.forwardr�)rarbrcr>rrr r�r�rr�rr`rer&r&rUr'r_s6325��������326	�rcr�)�FNetForTokenClassificationcr�r)r�rRrUr&r'r>�s327z#FNetForTokenClassification.__init__NrZr:r7r[r�r�r�r�cCs�|dur|n|jj}|j||||||d�}|d}	|�|	�}	|�|	�}328d}|dur9t�}||329�d|j�|�d��}|sO|330f|dd�}
|durM|f|
S|
St||331|j	d�S)z�332        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):333            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.334        Nr�rr8rjr�)335rTr�r�rLrrr�rrru)rSrZr:r7r[r�r�r�rvr�r�r�r�rr&r&r'r`�s(�	336337z"FNetForTokenClassification.forwardr�)rarbrcr>rrr r�r�rr�rr`rer&r&rUr'r�s6��������338	�rcs�eZdZ�fdd�Ze								ddeejdeejdeejdeejdeejd	eejd339eedeede	e340effd
d��Z�Z
S)�FNetForQuestionAnsweringcs<t��|�|j|_t|�|_t�|j|j�|_|�	�dSr))341r=r>rr�r�rrHrA�342qa_outputsr�rRrUr&r'r>�s343344z!FNetForQuestionAnswering.__init__NrZr:r7r[�start_positions�
end_positionsr�r�r�c	Cs>|dur|n|jj}|j||||||d�}	|	d}345|�|346�}|jddd�\}}
|�d���}|
�d���}
d}|dur|durt|���dkrL|�d�}t|���dkrY|�d�}|�d�}|�	d|�}|�	d|�}t347|d�}|||�}||
|�}||d}|s�||
f|	dd�}|dur�|f|S|St|||
|	jd�S)	Nr�rrr8rk)Zignore_indexrj)r��start_logits�348end_logitsru)
rTr�r�r�splitr�349contiguous�lenrP�clamprrru)rSrZr:r7r[r	r350r�r�rvr�r�rrr�Z
ignored_indexr�Z351start_lossZend_lossrr&r&r'r`sB�	352353354355356357358359�z FNetForQuestionAnswering.forwardr�)rarbrcr>rrr r�r�rr�rr`rer&r&rUr'r�s<��������	�360 361�r)362r�rr�r�rr�rr�r�r�)Irdr��dataclassesr�	functoolsr�typingrrr rZtorch.nnrrr	�utilsrrZscipyr
ZactivationsrZmodeling_layersrZmodeling_outputsrrrrrrrrrZmodeling_utilsrZ
pytorch_utilsrrZconfiguration_fnetrZ363get_loggerra�loggerr(r*r2�Moduler3rgrxr~r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�rrr�__all__r&r&r&r'�<module>s�,364	=&365�e�Y>�U�I[9D
Aluode/PerceptionLabPortable · CoolFace