CoolFace
Apppublic

Aluode/PerceptionLabPortable

sourceHugging Faceupdated 9mo agoView on Hugging Face
0likes
rootpython_embedLibsite-packagestransformersmodelsbert__pycache__modeling_bert.cpython-310.pyc
modeling_bert.cpython-310.pyc501 linesDownload Raw Back to __pycache__
1o

2.�Yi�1�@sXdZddlZddlZddlZddlmZddlmZmZddl	Z	ddl	m3Z4ddlmZm
Z
mZddlmZdd	lmZmZmZdd5lmZddlmZmZddlmZdd
lmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%ddl&m'Z'ddl(m)Z)m*Z*m+Z+ddl,m-Z-m.Z.m/Z/ddl0m1Z1ddl2m3Z3e/�4e5�Z6dd�Z7Gdd�de6j8�Z9Gdd�de7j8�Z:Gdd�de:�Z;Gdd�de8j8�Z<e:e;d�Z=Gdd �d e9j8�Z>Gd!d"�d"e10j8�Z?Gd#d$�d$e11j8�Z@Gd%d&�d&e�ZAGd'd(�d(e12j8�ZBGd)d*�d*e13j8�ZCGd+d,�d,e14j8�ZDGd-d.�d.e15j8�ZEGd/d0�d0e16j8�ZFGd1d2�d2e17j8�ZGGd3d4�d4e18j8�ZHe.Gd5d6�d6e'��ZIee.d7d8�Gd9d:�d:e-���ZJe.d;d8�Gd<d=�d=eI��ZKe.d>d8�Gd?d@�d@eI��ZLe.dAd8�GdBdC�dCeIe��ZMe.GdDdE�dEeI��ZNe.dFd8�GdGdH�dHeI��ZOe.dId8�GdJdK�dKeI��ZPe.GdLdM�dMeI��ZQe.GdNdO�dOeI��ZRe.GdPdQ�dQeI��ZSgdR�ZTdS)SzPyTorch BERT model.�N)�	dataclass)�Optional�Union)�nn)�BCEWithLogitsLoss�CrossEntropyLoss�MSELoss�)�ACT2FN)�Cache�DynamicCache�EncoderDecoderCache)�GenerationMixin)�#_prepare_4d_attention_mask_for_sdpa�*_prepare_4d_causal_attention_mask_for_sdpa)�GradientCheckpointingLayer)	�)BaseModelOutputWithPastAndCrossAttentions�,BaseModelOutputWithPoolingAndCrossAttentions�!CausalLMOutputWithCrossAttentions�MaskedLMOutput�MultipleChoiceModelOutput�NextSentencePredictorOutput�QuestionAnsweringModelOutput�SequenceClassifierOutput�TokenClassifierOutput)�PreTrainedModel)�apply_chunking_to_forward� find_pruneable_heads_and_indices�prune_linear_layer)�ModelOutput�auto_docstring�logging)�deprecate_kwarg�)�19BertConfigcCs�zddl}ddl}ddl}Wn
tyt�d��wtj�|�}t�	d|���|j20�|�}g}g}	|D] \}21}t�	d|22�d|���|j23�||24�}|�
|25�|	�
|�q6t||	�D]�\}26}|27�d�}28tdd	�|29D��rzt�	d30d�|31����q\|}
|32D]|}|�d|�r�|�d|�}n|g}|dd
ks�|ddkr�t|
d�}
nH|ddks�|ddkr�t|
d�}
n6|ddkr�t|
d�}
n*|ddkr�t|
d�}
nz	t|
|d�}
Wnty�t�	d33d�|34����Yq~wt|�dkr�t|d�}|
|}
q~|dd�dk�r35t|
d�}
n36|d
k�r|�|�}z|
j|jk�r)td|
j�d|j�d���Wnt�yC}z|j|
j|jf7_�d}~wwt�	d|37���t�|�|
_q\|S)z'Load tf checkpoints in a pytorch model.rNz�Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.z&Converting TensorFlow checkpoint from zLoading TF weight z with shape �/css�|]}|dvVqdS))Zadam_vZadam_mZAdamWeightDecayOptimizerZAdamWeightDecayOptimizer_1Zglobal_stepN�)�.0�nr&r&��E:\DocsHouse\542 percep lab latest\PerceptionLab\PerceptionLab_Portable\python_embed\Lib\site-packages\transformers/models/bert/modeling_bert.py�	<genexpr>Ss38��39�z*load_tf_weights_in_bert.<locals>.<genexpr>z	Skipping z
[A-Za-z]+_\d+z_(\d+)Zkernel�gamma�weightZoutput_bias�beta�biasZoutput_weightsZsquad�40classifier�r#i����Z_embeddingszPointer shape z and array shape z mismatchedzInitialize PyTorch weight )�re�numpyZ41tensorflow�ImportError�logger�error�os�path�abspath�info�trainZlist_variablesZ
load_variable�append�zip�split�any�join�	fullmatch�getattr�AttributeError�len�int�	transpose�shape�42ValueError�args�torchZ43from_numpy�data)�model�configZtf_checkpoint_pathr1�np�tfZtf_pathZ	init_vars�namesZarrays�namerF�array�pointerZm_nameZscope_names�num�er&r&r)�load_tf_weights_in_bert6s���4445���4647���rUc
sheZdZdZ�fdd�Z					ddeejdeejdeejd	eejd48e	dej49fdd
�Z�ZS)�BertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.cs�t���tj|j|j|jd�|_t�|j|j�|_	t�|j50|j�|_tj|j|j
d�|_t�|j�|_t|dd�|_|jdt�|j��d�dd�|jd	tj|j��tjd51�dd�dS)N)�padding_idx��eps�position_embedding_type�absolute�position_ids)r#�����F)�52persistent�token_type_ids��dtype)�super�__init__r�	Embedding�53vocab_size�hidden_size�pad_token_id�word_embeddings�max_position_embeddings�position_embeddingsZtype_vocab_size�token_type_embeddings�	LayerNorm�layer_norm_eps�Dropout�hidden_dropout_prob�dropoutrArZZregister_bufferrI�arange�expand�zerosr\�size�long��selfrL��	__class__r&r)rc�s54�55�zBertEmbeddings.__init__Nr�	input_idsr_r\�
inputs_embeds�past_key_values_length�returnc
Cs�|dur	|��}n|��dd�}|d}|dur&|jdd�|||�f}|durPt|d�rE|jdd�d|�f}|�|d|�}	|	}ntj|tj|jjd�}|durY|�	|�}|�56|�}57||58}|jdkrp|�|�}||7}|�
|�}|�|�}|S)Nr]r#r_r�ra�devicer[)rtr\�hasattrr_rrrIrsrurrhrkrZrjrlrp)
rwrzr_r\r{r|�input_shape�59seq_length�buffered_token_type_ids� buffered_token_type_ids_expandedrk�60embeddingsrjr&r&r)�forward�s,6162636465666768zBertEmbeddings.forward)NNNNr)
�__name__�69__module__�__qualname__�__doc__rcrrIZ70LongTensor�FloatTensorrD�Tensorr��
__classcell__r&r&rxr)rVs*�������rVcs�eZdZd�fdd�	Zedddd�						dd	ejd71eejdeejdeejdee	d
ee72deejdeejfdd��Z�Z
S)�BertSelfAttentionNcst���|j|jdkrt|d�std|j�d|j�d���|j|_t|j|j�|_|j|j|_t	�73|j|j�|_t	�74|j|j�|_t	�75|j|j�|_
t	�|j�|_|p\t|dd�|_|jdksh|jd	kry|j|_t	�d76|jd|j�|_|j|_||_dS)NrZembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (�)rZr[�relative_key�relative_key_queryr0r#)rbrcrf�num_attention_headsr�rGrD�attention_head_size�
all_head_sizer�Linear�query�key�valuern�attention_probs_dropout_probrprArZrird�distance_embedding�77is_decoder�	layer_idx�rwrLrZr�rxr&r)rc�s,7879���80zBertSelfAttention.__init__�past_key_value�past_key_values�4.58��new_name�versionF�
hidden_states�attention_mask�	head_mask�encoder_hidden_states�output_attentions�cache_positionr}cCs|j\}}	}81|�|�}|�|d|j|j��dd�}d}|du}
|dur;t|t�r9|j�	|j82�}|
r5|j}n|j}n|}|
r?|n|}|
rX|durX|rX|j
|j83j}|j
|j84j}nJ|�|�}|�|d|j|j��dd�}|�|�}|�|d|j|j��dd�}|dur�|
s�|nd}|�|||j85d|i�\}}|
r�t|t�r�d|j|j86<t�||�dd��}|jdks�|jd	k�r4|jd|jd}}|dur�tj|dtj|jd87��dd�}ntj|tj|jd88��dd�}tj|tj|jd89��dd�}||}|�||jd�}|j|jd�}|jdk�rt�d||�}||}n|jd	k�r4t�d||�}t�d
||�}|||}|t� |j�}|du�rE||}t!j"j#|dd�}|�$|�}|du�r[||}t�||�}|�%dddd��&�}|�'�dd�|j(f}|�|�}||fS)Nr]r#r0Fr�T�����r�r�r~r`zbhld,lrd->bhlrzbhrd,lrd->bhlr��dimrr	))rFr��viewr�r�rE�90isinstancer
�91is_updated�getr��cross_attention_cache�self_attention_cache�layers�keys�valuesr�r��updaterI�matmulrZZtensorrurrqr�ri�toraZeinsum�math�sqrtr�92functionalZsoftmaxrpZpermute�93contiguousrtr�)rwr�r�r�r�r�r�r��94batch_sizer��_�query_layerr��is_cross_attention�curr_past_key_value�current_states�	key_layer�value_layerZattention_scoresZquery_lengthZ95key_lengthZposition_ids_lZposition_ids_rZdistanceZpositional_embeddingZrelative_position_scoresZrelative_position_scores_queryZrelative_position_scores_keyZattention_probsZ
context_layerZnew_context_layer_shaper&r&r)r��s�96�9798�99����100101102103104zBertSelfAttention.forward�NN�NNNNFN�r�r�r�rcr"rIr�rr�r�bool�tupler�r�r&r&rxr)r��s4��������	�r�cs�eZdZd�fdd�	Zedddd�						dd	ejd105eejdeejdeejdee	d
ee106deejdeejf�fdd�
�Z�Z
S)�BertSdpaSelfAttentionNcst�j|||d�|j|_dS�N�rZr�)rbrcr��dropout_probr�rxr&r)rcEszBertSdpaSelfAttention.__init__r�r�r�r�Fr�r�r�r�r�r�r}c	s�|jdks|s|durt�d�t��|||||||�S|��\}}	}107|�|��|d|j|j	��108dd�}d}|du}
|
r>|n|}|dur\t|t�rZ|j
�|j�}|
rV|j}n|j}n|}|
r`|n|}|
ry|dury|ry|j|jj}|j|jj}nF|�|��|d|j|j	��109dd�}|�|��|d|j|j	��110dd�}|dur�|
s�|nd}|�|||jd|i�\}}|
r�t|t�r�d|j
|j<|jo�|
o�|duo�|	dk}tjjj|||||jr�|jnd	|d111�}|�112dd�}|�||	|j �}|dfS)Nr[a�BertSdpaSelfAttention is used but `torch.nn.functional.scaled_dot_product_attention` does not support non-absolute `position_embedding_type` or `output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.r]r#r0Fr�T�)Z	attn_maskZ	dropout_p�	is_causal)!rZr4�warning_oncerbr�rtr�r�r�r�rEr�r
r�r�r�r�r�r�r�r�r�r�r�r�rIrr�Zscaled_dot_product_attention�trainingr�Zreshaper�)rwr�r�r�r�r�r�r�Zbsz�tgt_lenr�r�r�r�r�r�r�r�r�Zattn_outputrxr&r)r�Jsr��113 �114����	zBertSdpaSelfAttention.forwardr�r�r�r&r&rxr)r�Ds4��������	�r�c�8eZdZ�fdd�Zdejdejdejfdd�Z�ZS)�BertSelfOutputcsBt���t�|j|j�|_tj|j|jd�|_t�|j	�|_115dS�NrX)rbrcrr�rf�denserlrmrnrorprvrxr&r)rc��116zBertSelfOutput.__init__r��input_tensorr}cC�&|�|�}|�|�}|�||�}|S�N�r�rprl�rwr�r�r&r&r)r���117118zBertSelfOutput.forward�r�r�r�rcrIr�r�r�r&r&rxr)r���$r�)�eager�sdpacs�eZdZd�fdd�	Zdd�Zedddd	�					119	ddejdeej	d
eej	deej	dee120deedeejdeejfdd��Z
�ZS)�
BertAttentionNcs6t���t|j|||d�|_t|�|_t�|_dSr�)	rbrc�BERT_SELF_ATTENTION_CLASSES�_attn_implementationrwr��output�set�pruned_headsr�rxr&r)rc�s121�122zBertAttention.__init__cCs�t|�dkrdSt||jj|jj|j�\}}t|jj|�|j_t|jj|�|j_t|jj	|�|j_	t|j123j|dd�|j124_|jjt|�|j_|jj|jj|j_|j�
|�|_dS)Nrr#r�)rCrrwr�r�r�rr�r�r�r�r�r��union)rw�heads�indexr&r&r)�prune_heads�s�zBertAttention.prune_headsr�r�r�r�Fr�r�r�r�r�r�r}c	Cs>|j|||||||d�}|�|d|�}	|	f|dd�}125|126S)N�r�r�r�r�r�r�rr#)rwr�)rwr�r�r�r�r�r�r�Zself_outputs�attention_output�outputsr&r&r)r��s�	zBertAttention.forwardr�r�)r�r�r�rcr�r"rIr�rr�rr�r�r�r�r&r&rxr)r��s6127��������	�r�c�2eZdZ�fdd�Zdejdejfdd�Z�ZS)�BertIntermediatecsDt���t�|j|j�|_t|jt	�rt128|j|_dS|j|_dSr�)rbrcrr�rf�intermediate_sizer�r��129hidden_act�strr130�intermediate_act_fnrvrxr&r)rc�s131132zBertIntermediate.__init__r�r}cC�|�|�}|�|�}|Sr�)r�r��rwr�r&r&r)r���133134zBertIntermediate.forwardr�r&r&rxr)r��sr�cr�)�135BertOutputcsBt���t�|j|j�|_tj|j|jd�|_t�	|j136�|_dSr�)rbrcrr�r�rfr�rlrmrnrorprvrxr&r)rcr�zBertOutput.__init__r�r�r}cCr�r�r�r�r&r&r)r�r�zBertOutput.forwardr�r&r&rxr)r�r�r�cs�eZdZd�fdd�	Zedddd�							dd	ejd137eejdeejdeejd
eejdee	dee138deejdeejfdd��Zdd�Z
�ZS)�	BertLayerNcsxt���|j|_d|_t||d�|_|j|_|j|_|jr0|js(t|�d���t|d|d�|_	t139|�|_t|�|_
dS)Nr#�r�z> should be used as a decoder model if cross attention is addedr[r�)rbrc�chunk_size_feed_forward�seq_len_dimr��	attentionr��add_cross_attentionrG�crossattentionr��intermediater�r��rwrLr�rxr&r)rcs140141zBertLayer.__init__r�r�r�r�Fr�r�r�r��encoder_attention_maskr�r�r}c		Cs�|j||||||d�}	|	d}142|	dd�}|jrA|durAt|d�s)td|�d���|j|143||||||d�}|d}144||dd�}t|j|j|j|145�}
|
f|}|S)N)r�r�r�r�r�rr#r�z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r�)	r�r�r�rGr�r�feed_forward_chunkr�r�)rwr�r�r�r�rr�r�r�Zself_attention_outputsr�r�Zcross_attention_outputs�layer_outputr&r&r)r�"s>�146147��	�148zBertLayer.forwardcCs|�|�}|�||�}|Sr�)rr�)rwr�Zintermediate_outputrr&r&r)rSs149zBertLayer.feed_forward_chunkr�)NNNNNFN)r�r�r�rcr"rIr�rr�rr�r�r�rr�r&r&rxr)r�s<��������	�150�0r�cs�eZdZd�fdd�	Z										ddejdeejdeejd	eejd151eejdeedee	d
ee	dee	dee	deejde152eejeffdd�Z
�ZS)�BertEncoderNcs:t����|_t��fdd�t�j�D��|_d|_dS)Ncsg|]}t�|d��qS)r�)r�)r'�i�rLr&r)�153<listcomp>]sz(BertEncoder.__init__.<locals>.<listcomp>F)	rbrcrLrZ154ModuleList�range�num_hidden_layers�layer�gradient_checkpointingrrxrr)rcZs155 156zBertEncoder.__init__FTr�r�r�r�rr��	use_cacher��output_hidden_states�return_dictr�r}cCs`|	rdnd}|r157dnd}
|r|jjrdnd}|jr%|jr%|r%t�d�d}|r<|jjr<|dur<tt|jd�t|jd��}|rQ|jjrQt	|t158�rQt�d�t�|�}t|j
�D]9\}}|	ra||f}|duri||nd}|||||||||d�}|d}|r�|
|df}
|jjr�||d	f}qV|	r�||f}|159s�t160d161d�||||
|fD��St||||
|d�S)
Nr&zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Frz�Passing a tuple of `past_key_values` is deprecated and will be removed in Transformers v4.58.0. You should pass an instance of `EncoderDecoderCache` instead, e.g. `past_key_values=EncoderDecoderCache.from_legacy_cache(past_key_values)`.)rr�r�r�rr#r0css�|]	}|dur|VqdSr�r&)r'�vr&r&r)r*�s���z&BertEncoder.forward.<locals>.<genexpr>)�last_hidden_stater�r��162attentions�cross_attentions)rLr�rr�r4r�r�r
rr�r�Zfrom_legacy_cache�	enumeraterr)rwr�r�r�r�rr�r
r�rrr�Zall_hidden_statesZall_self_attentionsZall_cross_attentionsrZlayer_moduleZlayer_head_maskZ
layer_outputsr&r&r)r�`sl��163164��165���zBertEncoder.forwardr�)166NNNNNNFFTN)r�r�r�rcrIr�rr�rr�rr�rr�r�r&r&rxr)rYsJ	��������	�167���
�rcr�)�168BertPoolercs*t���t�|j|j�|_t��|_dSr�)rbrcrr�rfr�ZTanh�169activationrvrxr&r)rc�s170zBertPooler.__init__r�r}cCs(|dd�df}|�|�}|�|�}|S)Nr)r�r)rwr�Zfirst_token_tensor�
pooled_outputr&r&r)r��s171172zBertPooler.forwardr�r&r&rxr)r�srcr�)�BertPredictionHeadTransformcsVt���t�|j|j�|_t|jt�rt	|j|_173n|j|_174tj|j|jd�|_dSr�)
rbrcrr�rfr�r�r�r�r175�transform_act_fnrlrmrvrxr&r)rc�s176z$BertPredictionHeadTransform.__init__r�r}cCs"|�|�}|�|�}|�|�}|Sr�)r�rrlr�r&r&r)r��s177178179z#BertPredictionHeadTransform.forwardr�r&r&rxr)r�s	rcs,eZdZ�fdd�Zdd�Zdd�Z�ZS)�BertLMPredictionHeadcsLt���t|�|_tj|j|jdd�|_t�	t180�|j��|_|j|j_dS)NF)r.)
rbrcr�	transformrr�rfre�decoder�	ParameterrIrsr.rvrxr&r)rc�s181182183zBertLMPredictionHead.__init__cCs|j|j_dSr�)r.r�rwr&r&r)�_tie_weights�sz!BertLMPredictionHead._tie_weightscCr�r�)rrr�r&r&r)r��r�zBertLMPredictionHead.forward)r�r�r�rcrr�r�r&r&rxr)r�s
rcr�)�BertOnlyMLMHeadcst���t|�|_dSr�)rbrcr�predictionsrvrxr&r)rc�s184zBertOnlyMLMHead.__init__�sequence_outputr}cC�|�|�}|Sr�)r!)rwr"�prediction_scoresr&r&r)r���185zBertOnlyMLMHead.forwardr�r&r&rxr)r �sr c�$eZdZ�fdd�Zdd�Z�ZS)�BertOnlyNSPHeadcst���t�|jd�|_dS�Nr0)rbrcrr�rf�seq_relationshiprvrxr&r)rc�s186zBertOnlyNSPHead.__init__cCr#r�)r))rwr�seq_relationship_scorer&r&r)r��r%zBertOnlyNSPHead.forward�r�r�r�rcr�r�r&r&rxr)r'�sr'cr&)�BertPreTrainingHeadscs(t���t|�|_t�|jd�|_dSr()rbrcrr!rr�rfr)rvrxr&r)rc�s187188zBertPreTrainingHeads.__init__cCs|�|�}|�|�}||fSr�)r!r))rwr"rr$r*r&r&r)r�s189190zBertPreTrainingHeads.forwardr+r&r&rxr)r,�sr,c@s.eZdZUeed<eZdZdZdZ	dd�Z191dS)�BertPreTrainedModelrL�bertTcCs�t|tj�r |jjjd|jjd�|jdur|jj�	�dSdSt|tj192�rC|jjjd|jjd�|jdurA|jj|j�	�dSdSt|tj�rX|jj�	�|jj�
d�dSt|t�re|jj�	�dSdS)zInitialize the weightsr�)�meanZstdNg�?)r�rr�r,rJZnormal_rLZinitializer_ranger.Zzero_rdrWrlZfill_r)rw�moduler&r&r)�
_init_weightss 193�194�195�z!BertPreTrainedModel._init_weightsN)r�r�r�r$�__annotations__rUZload_tf_weightsZbase_model_prefixZsupports_gradient_checkpointingZ_supports_sdpar1r&r&r&r)r-196s197r-z0198    Output type of [`BertForPreTraining`].199    )Zcustom_introc@steZdZUdZdZeejed<dZ	eejed<dZ200eejed<dZeeejed<dZ
eeejed<dS)�BertForPreTrainingOutputa�201    loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):202        Total loss as the sum of the masked language modeling loss and the next sequence prediction203        (classification) loss.204    prediction_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):205        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).206    seq_relationship_logits (`torch.FloatTensor` of shape `(batch_size, 2)`):207        Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation208        before SoftMax).209    N�loss�prediction_logits�seq_relationship_logitsr�r)r�r�r�r�r4rrIr�r2r5r6r�r�rr&r&r&r)r3%s210r3a211212    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of213    cross-attention is added between the self-attention layers, following the architecture described in [Attention is214    all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,215    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.216 217    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set218    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and219    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.220    c"s�eZdZddgZd�fdd�	Zdd�Zdd	�Zd221d�Ze														dd
e	e222jde	e223jde	e224jde	e225jde	e226jde	e227jde	e228jde	e229jde	ede	e
de	e
de	e
de	e
de	e230jdeee231jeffdd��Z�ZS) �	BertModelrVr�TcsTt��|�||_t|�|_t|�|_|rt|�nd|_|j	|_232|j|_|��dS)zv233        add_pooling_layer (bool, *optional*, defaults to `True`):234            Whether to add a pooling layer235        N)
rbrcrLrVr�r�encoderr�poolerr��attn_implementationrZ�	post_init)rwrL�add_pooling_layerrxr&r)rcMs236237zBertModel.__init__cCs|jjSr��r�rhrr&r&r)�get_input_embeddings`szBertModel.get_input_embeddingscCs||j_dSr�r=)rwr�r&r&r)�set_input_embeddingscszBertModel.set_input_embeddingscCs*|��D]\}}|jj|j�|�qdS)z�238        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base239        class PreTrainedModel240        N)�itemsr8rr�r�)rwZheads_to_prunerr�r&r&r)�_prune_headsfs�zBertModel._prune_headsNrzr�r_r\r�r{r�rr�r
r�rrr�r}c!
Cs�|dur|n|jj}|dur|n|jj}|
dur|
n|jj}
|jjr-|241dur(|242n|jj}243nd}244|dur;|dur;td��|durJ|�||�|��}n|durW|��dd�}ntd��|\}}|durf|j	n|j	}d}|	dur�t245|	t�s}|	ddjdn|	�
�}|dur�t|jd�r�|jjdd�d|�f}|�||�}|}n	tj|tj|d�}|j|||||d	�}|dur�tj|||f|d246�}|jdko�|jdko�|duo�|}|r�|��d
kr�|jjr�t||||�}nt||j|d�}n|�||�}|jj�r0|du�r0|��\}}}||f}|du�rtj||d247�}|�r*|��d
k�r*t||j|d�}n|�|�}nd}|�||jj�}|j ||||||	|248|||
|d�}|d}|j!du�rY|�!|�nd} |
�sh|| f|dd�St"|| |j#|j$|j%|j&d�S)NFzDYou cannot specify both input_ids and inputs_embeds at the same timer]z5You have to specify either input_ids or inputs_embedsrr�r_r~)rzr\r_r{r|)rr�r[r0)r�)249r�r�r�rr�r
r�rrr�r#)rZ
pooler_outputr�r�rr)'rLr�r�use_return_dictr�r
rGZ%warn_if_padding_and_no_attention_maskrtrr�rrFZget_seq_lengthr�r�r_rrrIrsruZonesr:rZr�rrraZget_extended_attention_maskZinvert_attention_maskZ
get_head_maskr250r8r9rr�r�rr)!rwrzr�r_r\r�r{r�rr�r
r�rrr�r�r�r�rr|r�r�Zembedding_outputZuse_sdpa_attention_masksZextended_attention_maskZencoder_batch_sizeZencoder_sequence_lengthr�Zencoder_hidden_shapeZencoder_extended_attention_maskZencoder_outputsr"rr&r&r)r�ns��251���252�����253��
�zBertModel.forward)T)NNNNNNNNNNNNNN)r�r�r�Z_no_split_modulesrcr>r?rAr rrIr�rr�rr�rr�r�r&r&rxr)r7>sh
��������	�254���
����r7z�255    Bert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next256    sentence prediction (classification)` head.257    cs�eZdZddgZ�fdd�Zdd�Zdd�Ze																						dd258ee	j259dee	j260dee	j261d
ee	j262dee	j263dee	j264dee	j265dee	j266deedeedeedee
e	j267effdd��Z�ZS)�BertForPreTraining�predictions.decoder.bias�cls.predictions.decoder.weightc�,t��|�t|�|_t|�|_|��dSr�)rbrcr7r.r,�clsr;rvrxr&r)rc�268269zBertForPreTraining.__init__cC�270|jjjSr��rGr!rrr&r&r)�get_output_embeddings�271z(BertForPreTraining.get_output_embeddingscC�||jj_|j|jj_dSr��rGr!rr.�rwZnew_embeddingsr&r&r)�set_output_embeddings�272z(BertForPreTraining.set_output_embeddingsNrzr�r_r\r�r{�labels�next_sentence_labelr�rrr}cCs�|dur|n|jj}|j|||||||	|273|d�	}|dd�\}
}|�|
|�\}}d}|durS|durSt�}||�d|jj�|�d��}||�dd�|�d��}||}|sj||f|dd�}|durh|f|S|St||||j|j	d�S)am274        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):275            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,276            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked),277            the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`278        next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):279            Labels for computing the next sequence prediction (classification) loss. Input should be a sequence280            pair (see `input_ids` docstring) Indices should be in `[0, 1]`:281 282            - 0 indicates sequence B is a continuation of sequence A,283            - 1 indicates sequence B is a random sequence.284 285        Example:286 287        ```python288        >>> from transformers import AutoTokenizer, BertForPreTraining289        >>> import torch290 291        >>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")292        >>> model = BertForPreTraining.from_pretrained("google-bert/bert-base-uncased")293 294        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")295        >>> outputs = model(**inputs)296 297        >>> prediction_logits = outputs.prediction_logits298        >>> seq_relationship_logits = outputs.seq_relationship_logits299        ```300        N�r�r_r\r�r{r�rrr0r])r4r5r6r�r)301rLrBr.rGrr�rer3r�r)rwrzr�r_r\r�r{rRrSr�rrr�r"rr$r*�302total_loss�loss_fct�masked_lm_loss�next_sentence_lossr�r&r&r)r�s<+��zBertForPreTraining.forward�NNNNNNNNNNN)r�r�r��_tied_weights_keysrcrKrPr rrIr�r�rr�r3r�r�r&r&rxr)rCsT	��������	�303���
�rCzP304    Bert Model with a `language modeling` head on top for CLM fine-tuning.305    c$seZdZddgZ�fdd�Zdd�Zdd�Ze																														dd306ee	j307dee	j308dee	j309d
ee	j310dee	j311dee	j312dee	j313dee	j314dee	j315deedeedeedeedeedee	j316de
ee	j317eff dd��Z�ZS)�BertLMHeadModelzcls.predictions.decoder.biasrEcs@t��|�|jst�d�t|dd�|_t|�|_|�	�dS)NzLIf you want to use `BertLMHeadModel` as a standalone, add `is_decoder=True.`F�r<�318rbrcr�r4�warningr7r.r rGr;rvrxr&r)rcvs319320zBertLMHeadModel.__init__cCrIr�rJrr&r&r)rK�rLz%BertLMHeadModel.get_output_embeddingscCrMr�rNrOr&r&r)rP�rQz%BertLMHeadModel.set_output_embeddingsNrzr�r_r\r�r{r�rrRr�r
r�rrr�r}cKs�|dur|n|jj}|	durd}|j|||||||||321|||
||d�}|d}|�|�}d}|	dur?|j||	|jjfi|��}|sU|f|dd�}|durS|f|S|St|||j|j|j	|j322d�S)a�323        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):324            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in325            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are326            ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`327        NF)
r�r_r\r�r{r�rr�r
r�rrr�rr0)r4�logitsr�r�rr)rLrBr.rGZ
loss_functionrerr�r�rr)rwrzr�r_r\r�r{r�rrRr�r
r�rrr�Zloss_kwargsr�r"r$Zlm_lossr�r&r&r)r��sF�328�zBertLMHeadModel.forward)NNNNNNNNNNNNNNN)r�r�r�rZrcrKrPr rrIr�rr�rr�rr�r�r&r&rxr)r[nsl��������	�329���
�����r[cs�eZdZddgZ�fdd�Zdd�Zdd�Ze																								dd330ee	j331dee	j332dee	j333d
ee	j334dee	j335dee	j336dee	j337dee	j338dee	j339deedeedeedee
e	j340effdd��Zddd�Zedefdd��Z�ZS)�BertForMaskedLMrDrEcs@t��|�|jrt�d�t|dd�|_t|�|_|�	�dS)NzkIf you want to use `BertForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.Fr\r]rvrxr&r)rc�s�341zBertForMaskedLM.__init__cCrIr�rJrr&r&r)rK�rLz%BertForMaskedLM.get_output_embeddingscCrMr�rNrOr&r&r)rP�rQz%BertForMaskedLM.set_output_embeddingsNrzr�r_r\r�r{r�rrRr�rrr}c

Cs�|dur|n|jj}|j|||||||||342||d�}
|
d}|�|�}d}|	dur:t�}||�d|jj�|	�d��}|sP|f|
dd�}|durN|f|S|St|||
j|
j	d�S)a�343        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):344            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,345            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the346            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`347        N)348r�r_r\r�r{r�rr�rrrr]r0�r4r_r�r)349rLrBr.rGrr�rerr�r)rwrzr�r_r\r�r{r�rrRr�rrr�r"r$rWrVr�r&r&r)r��s:�350�zBertForMaskedLM.forwardcKs~|j}|d}|jjdurtd��tj||�|jddf�gdd�}tj|df|jjtj|j	d�}tj||gdd�}||d�S)Nrz.The PAD token should be defined for generationr#r]r�r~)rzr�)351rFrLrgrGrI�catZ	new_zeros�fullrur)rwrzr�Zmodel_kwargsr�Zeffective_batch_sizeZdummy_tokenr&r&r)�prepare_inputs_for_generation!s"�352z-BertForMaskedLM.prepare_inputs_for_generationcCsdS)z�353        Legacy correction: BertForMaskedLM can't call `generate()` from `GenerationMixin`, even though it has a354        `prepare_inputs_for_generation` method.355        Fr&)rGr&r&r)�can_generate1szBertForMaskedLM.can_generate)NNNNNNNNNNNNr�)r�r�r�rZrcrKrPr rrIr�r�rr�rr�rd�classmethodrer�r&r&rxr)r`�s`��������	�356���
��3579r`zT358    Bert Model with a `next sentence prediction (classification)` head on top.359    c�eZdZ�fdd�Ze										ddeejdeejdeejdeejdeejd	eejd360eejdeedeed
eede	e361ejeffdd��Z�Z
S)�BertForNextSentencePredictioncrFr�)rbrcr7r.r'rGr;rvrxr&r)rc@rHz&BertForNextSentencePrediction.__init__Nrzr�r_r\r�r{rRr�rrr}cKs�d|vrt�dt�|�d�}|362dur|363n|jj}364|j||||||||	|365d�	}|d}
|�|
�}d}|durEt�}||�	dd�|�	d��}|366s[|f|dd�}|durY|f|S|St367|||j|jd�S)	a�368        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):369            Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair370            (see `input_ids` docstring). Indices should be in `[0, 1]`:371 372            - 0 indicates sequence B is a continuation of sequence A,373            - 1 indicates sequence B is a random sequence.374 375        Example:376 377        ```python378        >>> from transformers import AutoTokenizer, BertForNextSentencePrediction379        >>> import torch380 381        >>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")382        >>> model = BertForNextSentencePrediction.from_pretrained("google-bert/bert-base-uncased")383 384        >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."385        >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."386        >>> encoding = tokenizer(prompt, next_sentence, return_tensors="pt")387 388        >>> outputs = model(**encoding, labels=torch.LongTensor([1]))389        >>> logits = outputs.logits390        >>> assert logits[0, 0] < logits[0, 1]  # next sentence was random391        ```392        rSzoThe `next_sentence_label` argument is deprecated and will be removed in a future version, use `labels` instead.NrTr#r]r0ra)
�warnings�warn�
FutureWarning�poprLrBr.rGrr�rr�r)rwrzr�r_r\r�r{rRr�rr�kwargsr�rZseq_relationship_scoresrXrVr�r&r&r)r�IsB*�393�394�z%BertForNextSentencePrediction.forward�395NNNNNNNNNN)r�r�r�rcr rrIr�r�rr�rr�r�r&r&rxr)rh:sH	��������	�396��
�rhz�397    Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled398    output) e.g. for GLUE tasks.399    crg)�BertForSequenceClassificationcsdt��|�|j|_||_t|�|_|jdur|jn|j}t�	|�|_400t�|j|j�|_
|��dSr�)rbrc�401num_labelsrLr7r.�classifier_dropoutrorrnrpr�rfr/r;�rwrLrqrxr&r)rc�s402�z&BertForSequenceClassification.__init__Nrzr�r_r\r�r{rRr�rrr}cCsr|403dur|404n|jj}405|j||||||||	|406d�	}|d}|�|�}|�|�}
d}|dur�|jjdurV|jdkr<d|j_n|jdkrR|jtj	ksM|jtj407krRd|j_nd|j_|jjdkrtt�}|jdkrn||
��|���}n+||
|�}n%|jjdkr�t
�}||
�d|j�|�d��}n|jjdkr�t�}||
|�}|408s�|
f|dd�}|dur�|f|S|St||
|j|jd	�S)409a�410        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):411            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,412            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If413            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).414        NrTr#Z415regressionZsingle_label_classificationZmulti_label_classificationr]r0ra)rLrBr.rpr/Zproblem_typerprarIrurDr�squeezerr�rrr�r)rwrzr�r_r\r�r{rRr�rrr�rr_r4rVr�r&r&r)r��sV�416417418419"420421422�z%BertForSequenceClassification.forwardrn)r�r�r�rcr rrIr�r�rr�rr�r�r&r&rxr)ro�sH��������	�423���rocrg)�BertForMultipleChoicecsTt��|�t|�|_|jdur|jn|j}t�|�|_t�	|j424d�|_|��dS)Nr#)
rbrcr7r.rqrorrnrpr�rfr/r;rrrxr&r)rc�s425�zBertForMultipleChoice.__init__Nrzr�r_r\r�r{rRr�rrr}cCsn|426dur|427n|jj}428|dur|jdn|jd}|dur%|�d|�d��nd}|dur4|�d|�d��nd}|durC|�d|�d��nd}|durR|�d|�d��nd}|dure|�d|�d�|�d��nd}|j||||||||	|429d�	}|d}
|�|
�}
|�|
�}|�d|�}d}|dur�t�}|||�}|430s�|f|dd�}|dur�|f|S|St	|||j431|jd�S)a[432        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):433            Indices of input sequence tokens in the vocabulary.434 435            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and436            [`PreTrainedTokenizer.__call__`] for details.437 438            [What are input IDs?](../glossary#input-ids)439        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):440            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,441            1]`:442 443            - 0 corresponds to a *sentence A* token,444            - 1 corresponds to a *sentence B* token.445 446            [What are token type IDs?](../glossary#token-type-ids)447        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):448            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,449            config.max_position_embeddings - 1]`.450 451            [What are position IDs?](../glossary#position-ids)452        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):453            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This454            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the455            model's internal embedding lookup matrix.456        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):457            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,458            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See459            `input_ids` above)460        Nr#r]r�rTr0ra)rLrBrFr�rtr.rpr/rrr�r)rwrzr�r_r\r�r{rRr�rrZnum_choicesr�rr_Zreshaped_logitsr4rVr�r&r&r)r�sL,���461462463�zBertForMultipleChoice.forwardrn)r�r�r�rcr rrIr�r�rr�rr�r�r&r&rxr)rt�sH
��������	�464���rtcrg)�BertForTokenClassificationcsbt��|�|j|_t|dd�|_|jdur|jn|j}t�|�|_	t�465|j|j�|_|�
�dS�NFr\)rbrcrpr7r.rqrorrnrpr�rfr/r;rrrxr&r)rcjs�z#BertForTokenClassification.__init__Nrzr�r_r\r�r{rRr�rrr}cCs�|466dur|467n|jj}468|j||||||||	|469d�	}|d}|�|�}|�|�}
d}|dur<t�}||
�d|j�|�d��}|470sR|
f|dd�}|durP|f|S|St||
|j	|j471d�S)z�472        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):473            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.474        NrTrr]r0ra)rLrBr.rpr/rr�rprr�r)rwrzr�r_r\r�r{rRr�rrr�r"r_r4rVr�r&r&r)r�xs8�475476�z"BertForTokenClassification.forwardrn)r�r�r�rcr rrIr�r�rr�rr�r�r&r&rxr)ruhsH��������	�477���rucs�eZdZ�fdd�Ze											ddeejdeejdeejdeejdeejd	eejd478eejdeejdeed
eedeede	e479ejeffdd��Z�Z
S)�BertForQuestionAnsweringcs@t��|�|j|_t|dd�|_t�|j|j�|_|�	�dSrv)480rbrcrpr7r.rr�rf�481qa_outputsr;rvrxr&r)rc�s482z!BertForQuestionAnswering.__init__Nrzr�r_r\r�r{�start_positions�
end_positionsr�rrr}cCsH|dur|n|jj}|j|||||||	|483|d�	}|d}
|�|
�}|jddd�\}}|�d���}|�d���}d}|dur�|dur�t|���dkrO|�d�}t|���dkr\|�d�}|�d�}|�	d|�}|�	d|�}t484|d�}|||�}|||�}||d}|s�||f|dd�}|dur�|f|S|St||||j|j
d�S)	NrTrr#r]r�)Zignore_indexr0)r4�start_logits�485end_logitsr�r)rLrBr.rxr=rsr�rCrt�clamprrr�r)rwrzr�r_r\r�r{ryrzr�rrr�r"r_r{r|rUZ
ignored_indexrVZ486start_lossZend_lossr�r&r&r)r��sP�487488489490491492493�z BertForQuestionAnswering.forwardrY)r�r�r�rcr rrIr�r�rr�rr�r�r&r&rxr)rw�sN494��������	�495���
�rw)r`rtrhrCrwrorur�r[r7r-rU)Ur�r�r6ri�dataclassesr�typingrrrIrZtorch.nnrrrZactivationsr496Zcache_utilsrrr
Z497generationrZmodeling_attn_mask_utilsrrZmodeling_layersrZmodeling_outputsrrrrrrrrrZmodeling_utilsrZ
pytorch_utilsrrr�utilsrr r!Zutils.deprecationr"Zconfiguration_bertr$Z498get_loggerr�r4rU�ModulerVr�r�r�r�r�r�r�r�rrrrr r'r,r-r3r7rCr[r`rhrortrurw�__all__r&r&r&r)�<module>s�,499I@h�6FZ500501��<�c�Zl�_�YjEM
Aluode/PerceptionLabPortable · CoolFace