CoolFace
Apppublic

seai2526-uniba-TheClouds/Code-Comment-Classification-Api

sourceHugging Facemitupdated 9mo agoView on Hugging Face
1likes
train.cpython-311.pyc56 linesDownload Raw Back to __pycache__
1�

2�*i7�	�|�ddlZddlZddlZddlZddlZddlZddlmZmZddl	m3Z4ejejd���ej
e��Zejddd	�5��d�Zedkr�ej��Ze�d
ed	���e�ded	���e���Zedd��5Zeje��Zddd��n#1swxYwYeej���Zeejejddej�dej��e���dSdS)�N�)�load_dataset_splits�parse_labels_column)�Datasetz4%(asctime)s - %(name)s - %(levelname)s - %(message)s)�level�formatzse4ai2526-uniba�	TheCloudsT)�6repo_owner�	repo_name�mlflowc�<
�td|�d|�d���t|��}||�d�}||�d�}t|��}t|��}tj|d���}tj|d���}	|dk�r�d	d7lm}8m}m}tj9d��tjd|�d
|�����5tjd|��tjd|��|10�
dd���}
|dVi|��}||
|||	ddd����}tjd|j��tjd|j��|���|���}|���D]\}}tj||���|j�|��tj�||�d�d���tj��ddd��n#1swxYwY�nk|dk�r@d	dl}d	dl}d	dlm}d	d lm}d	d!lm }d	d"l!m"}tj11d#��tjd|�d
|�����5tjd|��tjd|��tj#|��tI|�%d$d%����|�%d&d��|�%d'd(��|�%d)d*��d+�}|}|d,|dVi|��fd-||dVd.d/d0�|����fg��}|d1}|�&|d2��}|�'||��|	d1}|�&|	d2��}|�(||��} tjd3| ��tSj*tRj+�,|��d4�5��|�-||�d6���tj.�||�d7��8��tj��ddd��n#1swxYwY�n$|d9k�rd(d:l/m0}!m1}"tj12d;��tjd|�d
|�����5tjd|��tjd|��tj#|��|!|d<d=||�2d>d?��|�2d@dA��|�2dBdC��|�2dDdE��|�2ddF��|�2dGdH��|�2dIdJ��|�2dKdL��|�2dMd��|�2dNd*���O��}#tf�4dP||#��|"|#��}|�5��}$tf�4dQ||$��|$���D]\}%}&tjdR|%��|&���tj��ddd��n#1swxYwYntmdS|�����tdT|�d
|�dU|����dS)Wz@Trains and saves a model for a specific language and model type.z$--- Starting training for language: z
 with model: z ---�_train�_testF)�preserve_index�setfitr)�SetFitModel�Trainer�TrainingArgumentszSetFit Trainingztrain-�-)�run_name�language�13model_typez-sentence-transformers/paraphrase-MiniLM-L6-v2zmulti-output)�multi_target_strategy�text�label)�combo�labels)�model�args�
train_dataset�eval_dataset�column_mapping�14num_epochs�num_iterations�
_setfit_modelztext-classification)�transformers_model�
artifact_path�taskN�
random_forest)�RandomForestClassifier)�TfidfVectorizer)�MultiOutputClassifier)�PipelinezRandom Forest Training�ngram_range)rr�max_features�min_dfr�max_dfg�?)r.r/r0r1�tfidf�clf�*�balanced)�random_state�class_weightrr�accuracyT)�exist_okz.joblib�_random_forest_model)�sk_modelr'�transformer)�TransformerConfig�TransformerTrainerzTransformer Training�data/rawzdata/processed/transformer�pretrained_model_namezmicrosoft/codebert-base�15max_length��16batch_size��lrg�h㈵��>��warmup_ratiog�������?�pos_weight_capg>@�	thresholdg�?�
preprocessing�preprocessing_factor)�lang�raw_data_dir�processed_data_dir�model_output_pathr@rArCrEr#rGrHrIrJrKz?Starting transformer training for language '%s' with config: %sz$Final transformer metrics for %s: %s�final_zUnsupported model_type: z17Model for z18 saved to �)7�printrrr�from_pandasrrrrr�set_experiment�	start_run�	log_param�from_pretrainedr#r$�train�evaluate�items�19log_metricr�save_pretrained�transformers�	log_model�end_run�numpy�joblib�sklearn.ensembler*�sklearn.feature_extraction.textr+�sklearn.multioutputr,�sklearn.pipeliner-�20log_params�tuple�pop�array�fit�score�os�makedirs�path�dirname�dump�sklearnr<r=r>�get�logger�info�run�21ValueError)'rLr�	data_pathrO�params�ds�train_df�eval_dfr r!rrrrr�trainer�eval_metrics�metric_name�metric_value�nprar*r+r,r-�tfidf_params�	rf_params�pipeline�X_train�y_train�X_test�y_testrkr=r>�cfg�metrics�name�values'                                       �E/home/kali/SwEn/TheClouds/codecommentclassification/modeling/train.py�train_modelr�se��	�22T��23T�24T�J�25T�26T�27T�U�U�U�	�Y�	'�	'�B��T�/�/�/�"�H��D���� �G�"�8�,�,�H�!�'�*�*�G��'���G�G�G�M��&�w�u�E�E�E�L��X���B�B�B�B�B�B�B�B�B�B���/�0�0�0�
�
�'C��'C�'C�z�'C�'C�
D�
D�
D� 	� 	���Z��.�.�.���\�:�6�6�6��/�/�?�&4�0���E�%�$�.�.�v�.�.�D��g���+�)�)/�7�C�C����G�
��\�4�?�;�;�;���-�t�/B�C�C�C��M�M�O�O�O�"�+�+�-�-�L�-9�-?�-?�-A�-A�
=�
=�)��\��!�+�|�<�<�<�<��M�)�)�*;�<�<�<���)�)�#4�!%�4�4�4�*�
*�
�
�
�28
�N����A 	� 	� 	� 	� 	� 	� 	� 	� 	� 	� 	���� 	� 	� 	� 	��D29��	&�	&������
�
�
�;�;�;�;�;�;�C�C�C�C�C�C�=�=�=�=�=�=�-�-�-�-�-�-���6�7�7�7�
�
�'C��'C�'C�z�'C�'C�
D�
D�
D�,	�,	���Z��.�.�.���\�:�6�6�6���f�%�%�%� %�V�Z�Z�
�v�%F�%F�G�G� &�30�31�>�4� @� @� �*�*�X�q�1�1� �*�*�X�s�3�3�	��L��I��x��o�o�=�=��=�=�>��-�-�2�2��-/�j���LU������32���H�$�G�,�G��h�h�}�X�6�7�7�G��L�L��'�*�*�*�!�'�*�F��X�X�l�8�4�5�5�F��N�N�6�6�2�2�E���j�%�0�0�0��K�����(9�:�:�T�J�J�J�J��K�K��%6�"?�"?�"?�@�@�@��N�$�$�!�D�1N�1N�1N�
%�
�
�
�
�N����Y,	�,	�,	�,	�,	�,	�,	�,	�,	�,	�,	����,	�,	�,	�,	��\33�}�	$�	$�	34�	35�	36�	37�	38�	39�	40�	41�42	��4�5�5�5�
�
�'C��'C�'C�z�'C�'C�
D�
D�
D�&	�&	���Z��.�.�.���\�:�6�6�6���f�%�%�%�#�#��'�#?�"3�&,�j�j�+�-F�'�'�"�:�:�l�C�8�8�!�:�:�l�B�7�7��:�:�d�D�)�)�!�:�:�l�A�6�6�#�Z�Z���<�<�%�z�z�*:�D�A�A� �*�*�[�#�6�6�$�j�j��%�@�@�%+�Z�Z�0F��%L�%L�!���C�&
�K�K�Q���
�
�
�)�(��-�-�G��k�k�m�m�G��K�K�>��g�N�N�N�&�}�}���
:�
:���e��!�/�4�/�/�5�9�9�9�9��N����M&	�&	�&	�&	�&	�&	�&	�&	�&	�&	�&	����&	�&	�&	�&	��R�@�J�@�@�A�A�A�	�43G�t�44G�45G�j�46G�47G�4E�48G�49G�H�H�H�H�Hs9�D:H�H�H�/GQ�Q�Q�F<Y&�&Y*�-Y*�__main__z--lang)�type�requiredz--model_typezparams.yaml�rr?zmodels/�/)rLrrwrOrx) �argparse�loggingrl�yaml�dagshubr�utilsrr�datasetsr�basicConfig�INFO�	getLogger�__name__rs�initr��ArgumentParser�parser�add_argument�str�50parse_argsr�open�f�	safe_load�51all_paramsr�copy�model_paramsrLrQ�r��<module>r�s����������	�	�	�	���������
�
�
�
�;�;�;�;�;�;�;�;����������52�,�A�����53��	�8�	$�	$��
���)�[��N�N�N�N�_I�_I�_I�J�z���
$�X�
$�
&�
&�F�54����s�T��:�:�:�55����S�4��@�@�@������D�	
��m�S�	!�	!�'�Q�#�T�^�A�&�&�56�'�'�'�'�'�'�'�'�'�'�'����'�'�'�'��d�o�.�3�3�5�5�L��K�
�Y��?��A�D�I�A�A���A�A���������s�	C&�&C*�-C*