{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "aa95efb4-71eb-4f3d-8799-c12831385900", "metadata": {}, "outputs": [], "source": [ "import os\n", "import pandas as pd\n", "import torch\n", "from torch import nn, optim\n", "from torch.utils.data import DataLoader, Dataset\n", "from transformers import AutoTokenizer, AutoModel, get_linear_schedule_with_warmup\n", "from sklearn.metrics import accuracy_score, f1_score, classification_report\n", "from sklearn.model_selection import StratifiedKFold, train_test_split\n", "import random\n", "import numpy as np\n", "import statistics" ] }, { "cell_type": "code", "execution_count": 2, "id": "3e6b7a75-f6d2-4fd7-968c-b16f9832d1ca", "metadata": {}, "outputs": [], "source": [ "#Mudanças principais:\n", "#1400\n", "#Modelo Bertimbau Large: Alterado o model_name para 'neuralmind/bert-large-portuguese-cased'.\n", "\n", "#LR= 3e-5.\n", "\n", "#Descongelamento das camadas: Parametrizamos o número de camadas finais do BERT a descongelar, via unfreeze_layers. Por exemplo, se definirmos unfreeze_layers=8, descongelamos as últimas 8 camadas.\n", "\n", "#Outros otimizadores e LR Schedulers: Mantemos o AdamW como otimizador principal, mas agora adicionamos um scheduler (get_linear_schedule_with_warmup do transformers) para ajustar a taxa de aprendizado durante o treino. Caso queira testar outro otimizador, basta substituir a linha do optimizador. Também deixamos comentado outro exemplo (SGD) para referência.\n", "#Para testar diferentes taxas de aprendizado, basta alterar learning_rate no código.\n", "#Para testar diferentes números de camadas a descongelar, altere unfreeze_layers.\n", "\n", "#4\n", "#processo de treinamento e avaliação várias vezes (uma para cada fold).\n", "#diminuindo épocas ou early stopping, se necessário.\n", "#O early stopping agora é feito com base no conjunto de validação interno a cada fold.\n", "#Esse processo é mais demorado, pois treinaremos o modelo K vezes.\n", "#Ajuste parâmetros (como número de épocas, taxa de aprendizado, etc.) conforme necessário." ] }, { "cell_type": "code", "execution_count": 3, "id": "e375e916-07a1-44f1-9675-8fb7eb8045f1", "metadata": {}, "outputs": [], "source": [ "# Semente para reprodutibilidade\n", "seed = 42\n", "random.seed(seed)\n", "np.random.seed(seed)\n", "torch.manual_seed(seed)\n", "if torch.cuda.is_available():\n", " torch.cuda.manual_seed_all(seed)" ] }, { "cell_type": "code", "execution_count": 4, "id": "1b511a6a-4c56-4335-8c15-660d3d146399", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Usando dispositivo: cuda\n" ] } ], "source": [ "# Configurações gerais\n", "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", "print(f'Usando dispositivo: {device}')\n", "\n", "model_name = 'neuralmind/bert-large-portuguese-cased' \n", "learning_rate = 3e-5\n", "unfreeze_layers = 4\n", "nclasses = 2\n", "nepochs = 5\n", "batch_size = 8\n", "batch_status = 32\n", "early_stop = 2\n", "max_length = 360\n", "write_path = 'C:/model_xlm_roberta_cv8'\n", "\n", "if not os.path.exists(write_path):\n", " os.makedirs(write_path)" ] }, { "cell_type": "code", "execution_count": 5, "id": "3a0a71c0-667a-421f-b324-fa857999aa6f", "metadata": {}, "outputs": [], "source": [ "# Carregar os dados\n", "data = pd.read_csv(\"DATAFRAME1400_augmented.csv\")" ] }, { "cell_type": "code", "execution_count": 6, "id": "ff7f8d91-5750-45f3-9a75-1240e5401757", "metadata": {}, "outputs": [], "source": [ "# Dataset Customizado\n", "class CustomDataset(Dataset):\n", " def __init__(self, data, tokenizer, max_length):\n", " self.data = data.reset_index(drop=True)\n", " self.tokenizer = tokenizer\n", " self.max_length = max_length\n", "\n", " def __len__(self):\n", " return len(self.data)\n", "\n", " def __getitem__(self, idx):\n", " text = self.data.iloc[idx]['text']\n", " label = self.data.iloc[idx]['contra']\n", " inputs = self.tokenizer(text, return_tensors='pt',\n", " padding='max_length', truncation=True,\n", " max_length=self.max_length)\n", " return {key: val.squeeze(0) for key, val in inputs.items()}, torch.tensor(label)\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 7, "id": "ac08faef-e9c9-496f-b850-6295f892c4e6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "=== Fold 1/5 ===\n", "Epoch: 0 [32/539]\tLoss: 0.806810\n", "Epoch: 0 [64/539]\tLoss: 0.893960\n", "Epoch: 0 [96/539]\tLoss: 0.640361\n", "Epoch: 0 [128/539]\tLoss: 0.615714\n", "Epoch: 0 [160/539]\tLoss: 0.570503\n", "Epoch: 0 [192/539]\tLoss: 0.630810\n", "Epoch: 0 [224/539]\tLoss: 0.643570\n", "Epoch: 0 [256/539]\tLoss: 0.768140\n", "Epoch: 0 [288/539]\tLoss: 1.057127\n", "Epoch: 0 [320/539]\tLoss: 0.569394\n", "Epoch: 0 [352/539]\tLoss: 0.381507\n", "Epoch: 0 [384/539]\tLoss: 0.306017\n", "Epoch: 0 [416/539]\tLoss: 0.685640\n", "Epoch: 0 [448/539]\tLoss: 0.419818\n", "Epoch: 0 [480/539]\tLoss: 0.364408\n", "Epoch: 0 [512/539]\tLoss: 0.748105\n", "Epoch 0 - Val F1: 0.8243, Val Accuracy: 0.8246\n", "Novo melhor modelo salvo.\n", "Epoch: 1 [32/539]\tLoss: 0.233392\n", "Epoch: 1 [64/539]\tLoss: 0.336354\n", "Epoch: 1 [96/539]\tLoss: 0.421595\n", "Epoch: 1 [128/539]\tLoss: 0.099084\n", "Epoch: 1 [160/539]\tLoss: 0.365469\n", "Epoch: 1 [192/539]\tLoss: 0.614397\n", "Epoch: 1 [224/539]\tLoss: 0.651654\n", "Epoch: 1 [256/539]\tLoss: 0.123179\n", "Epoch: 1 [288/539]\tLoss: 0.272575\n", "Epoch: 1 [320/539]\tLoss: 0.270919\n", "Epoch: 1 [352/539]\tLoss: 0.199035\n", "Epoch: 1 [384/539]\tLoss: 0.386721\n", "Epoch: 1 [416/539]\tLoss: 0.824670\n", "Epoch: 1 [448/539]\tLoss: 0.282997\n", "Epoch: 1 [480/539]\tLoss: 0.184333\n", "Epoch: 1 [512/539]\tLoss: 0.038649\n", "Epoch 1 - Val F1: 0.8559, Val Accuracy: 0.8559\n", "Novo melhor modelo salvo.\n", "Epoch: 2 [32/539]\tLoss: 0.168970\n", "Epoch: 2 [64/539]\tLoss: 0.351753\n", "Epoch: 2 [96/539]\tLoss: 0.034803\n", "Epoch: 2 [128/539]\tLoss: 0.011378\n", "Epoch: 2 [160/539]\tLoss: 0.508281\n", "Epoch: 2 [192/539]\tLoss: 0.069164\n", "Epoch: 2 [224/539]\tLoss: 0.061568\n", "Epoch: 2 [256/539]\tLoss: 0.309656\n", "Epoch: 2 [288/539]\tLoss: 0.024734\n", "Epoch: 2 [320/539]\tLoss: 0.132665\n", "Epoch: 2 [352/539]\tLoss: 0.025687\n", "Epoch: 2 [384/539]\tLoss: 0.571691\n", "Epoch: 2 [416/539]\tLoss: 0.420970\n", "Epoch: 2 [448/539]\tLoss: 0.036176\n", "Epoch: 2 [480/539]\tLoss: 0.161503\n", "Epoch: 2 [512/539]\tLoss: 0.021420\n", "Epoch 2 - Val F1: 0.9186, Val Accuracy: 0.9186\n", "Novo melhor modelo salvo.\n", "Epoch: 3 [32/539]\tLoss: 0.060905\n", "Epoch: 3 [64/539]\tLoss: 0.033918\n", "Epoch: 3 [96/539]\tLoss: 0.047984\n", "Epoch: 3 [128/539]\tLoss: 0.008567\n", "Epoch: 3 [160/539]\tLoss: 0.057391\n", "Epoch: 3 [192/539]\tLoss: 0.028192\n", "Epoch: 3 [224/539]\tLoss: 0.087906\n", "Epoch: 3 [256/539]\tLoss: 0.048106\n", "Epoch: 3 [288/539]\tLoss: 0.020358\n", "Epoch: 3 [320/539]\tLoss: 0.018573\n", "Epoch: 3 [352/539]\tLoss: 0.004970\n", "Epoch: 3 [384/539]\tLoss: 0.144033\n", "Epoch: 3 [416/539]\tLoss: 0.081381\n", "Epoch: 3 [448/539]\tLoss: 0.008459\n", "Epoch: 3 [480/539]\tLoss: 0.298734\n", "Epoch: 3 [512/539]\tLoss: 0.032544\n", "Epoch 3 - Val F1: 0.9207, Val Accuracy: 0.9207\n", "Novo melhor modelo salvo.\n", "Epoch: 4 [32/539]\tLoss: 0.011373\n", "Epoch: 4 [64/539]\tLoss: 0.010223\n", "Epoch: 4 [96/539]\tLoss: 0.008893\n", "Epoch: 4 [128/539]\tLoss: 0.051655\n", "Epoch: 4 [160/539]\tLoss: 0.060906\n", "Epoch: 4 [192/539]\tLoss: 0.005525\n", "Epoch: 4 [224/539]\tLoss: 0.105270\n", "Epoch: 4 [256/539]\tLoss: 0.003194\n", "Epoch: 4 [288/539]\tLoss: 0.011297\n", "Epoch: 4 [320/539]\tLoss: 0.430329\n", "Epoch: 4 [352/539]\tLoss: 0.007904\n", "Epoch: 4 [384/539]\tLoss: 0.029006\n", "Epoch: 4 [416/539]\tLoss: 0.005053\n", "Epoch: 4 [448/539]\tLoss: 0.016824\n", "Epoch: 4 [480/539]\tLoss: 0.021264\n", "Epoch: 4 [512/539]\tLoss: 0.079554\n", "Epoch 4 - Val F1: 0.9332, Val Accuracy: 0.9332\n", "Novo melhor modelo salvo.\n", "Desempenho no conjunto de teste desta dobra:\n", " precision recall f1-score support\n", "\n", " 0 0.92 0.95 0.93 627\n", " 1 0.94 0.91 0.92 570\n", "\n", " accuracy 0.93 1197\n", " macro avg 0.93 0.93 0.93 1197\n", "weighted avg 0.93 0.93 0.93 1197\n", "\n", "F1 (teste): 0.9289, Accuracy (teste): 0.9290\n", "\n", "=== Fold 2/5 ===\n", "Epoch: 0 [32/539]\tLoss: 0.698321\n", "Epoch: 0 [64/539]\tLoss: 0.653913\n", "Epoch: 0 [96/539]\tLoss: 0.659919\n", "Epoch: 0 [128/539]\tLoss: 0.683509\n", "Epoch: 0 [160/539]\tLoss: 0.634293\n", "Epoch: 0 [192/539]\tLoss: 0.671241\n", "Epoch: 0 [224/539]\tLoss: 0.625201\n", "Epoch: 0 [256/539]\tLoss: 0.452264\n", "Epoch: 0 [288/539]\tLoss: 0.487493\n", "Epoch: 0 [320/539]\tLoss: 0.263387\n", "Epoch: 0 [352/539]\tLoss: 0.548454\n", "Epoch: 0 [384/539]\tLoss: 0.539689\n", "Epoch: 0 [416/539]\tLoss: 0.267489\n", "Epoch: 0 [448/539]\tLoss: 0.373991\n", "Epoch: 0 [480/539]\tLoss: 0.328955\n", "Epoch: 0 [512/539]\tLoss: 0.192182\n", "Epoch 0 - Val F1: 0.8142, Val Accuracy: 0.8142\n", "Novo melhor modelo salvo.\n", "Epoch: 1 [32/539]\tLoss: 0.304935\n", "Epoch: 1 [64/539]\tLoss: 0.417642\n", "Epoch: 1 [96/539]\tLoss: 0.266215\n", "Epoch: 1 [128/539]\tLoss: 0.528966\n", "Epoch: 1 [160/539]\tLoss: 0.147393\n", "Epoch: 1 [192/539]\tLoss: 0.070752\n", "Epoch: 1 [224/539]\tLoss: 0.301449\n", "Epoch: 1 [256/539]\tLoss: 0.427771\n", "Epoch: 1 [288/539]\tLoss: 0.160852\n", "Epoch: 1 [320/539]\tLoss: 0.351679\n", "Epoch: 1 [352/539]\tLoss: 0.058495\n", "Epoch: 1 [384/539]\tLoss: 0.198829\n", "Epoch: 1 [416/539]\tLoss: 0.036262\n", "Epoch: 1 [448/539]\tLoss: 0.270694\n", "Epoch: 1 [480/539]\tLoss: 0.055876\n", "Epoch: 1 [512/539]\tLoss: 0.377879\n", "Epoch 1 - Val F1: 0.9101, Val Accuracy: 0.9102\n", "Novo melhor modelo salvo.\n", "Epoch: 2 [32/539]\tLoss: 0.088281\n", "Epoch: 2 [64/539]\tLoss: 0.170918\n", "Epoch: 2 [96/539]\tLoss: 0.131975\n", "Epoch: 2 [128/539]\tLoss: 0.112684\n", "Epoch: 2 [160/539]\tLoss: 0.087198\n", "Epoch: 2 [192/539]\tLoss: 0.022119\n", "Epoch: 2 [224/539]\tLoss: 0.111156\n", "Epoch: 2 [256/539]\tLoss: 0.039095\n", "Epoch: 2 [288/539]\tLoss: 0.035981\n", "Epoch: 2 [320/539]\tLoss: 0.063993\n", "Epoch: 2 [352/539]\tLoss: 0.289066\n", "Epoch: 2 [384/539]\tLoss: 0.007462\n", "Epoch: 2 [416/539]\tLoss: 0.038028\n", "Epoch: 2 [448/539]\tLoss: 0.245104\n", "Epoch: 2 [480/539]\tLoss: 0.161102\n", "Epoch: 2 [512/539]\tLoss: 0.359799\n", "Epoch 2 - Val F1: 0.9352, Val Accuracy: 0.9353\n", "Novo melhor modelo salvo.\n", "Epoch: 3 [32/539]\tLoss: 0.017929\n", "Epoch: 3 [64/539]\tLoss: 0.236215\n", "Epoch: 3 [96/539]\tLoss: 0.008162\n", "Epoch: 3 [128/539]\tLoss: 0.002611\n", "Epoch: 3 [160/539]\tLoss: 0.014606\n", "Epoch: 3 [192/539]\tLoss: 0.010964\n", "Epoch: 3 [224/539]\tLoss: 0.009766\n", "Epoch: 3 [256/539]\tLoss: 0.229709\n", "Epoch: 3 [288/539]\tLoss: 0.002626\n", "Epoch: 3 [320/539]\tLoss: 0.079086\n", "Epoch: 3 [352/539]\tLoss: 0.008535\n", "Epoch: 3 [384/539]\tLoss: 0.038777\n", "Epoch: 3 [416/539]\tLoss: 0.023151\n", "Epoch: 3 [448/539]\tLoss: 0.011855\n", "Epoch: 3 [480/539]\tLoss: 0.003287\n", "Epoch: 3 [512/539]\tLoss: 0.050024\n", "Epoch 3 - Val F1: 0.9309, Val Accuracy: 0.9311\n", "Epoch: 4 [32/539]\tLoss: 0.009518\n", "Epoch: 4 [64/539]\tLoss: 0.005575\n", "Epoch: 4 [96/539]\tLoss: 0.023808\n", "Epoch: 4 [128/539]\tLoss: 0.006816\n", "Epoch: 4 [160/539]\tLoss: 0.001851\n", "Epoch: 4 [192/539]\tLoss: 0.006231\n", "Epoch: 4 [224/539]\tLoss: 0.028360\n", "Epoch: 4 [256/539]\tLoss: 0.002173\n", "Epoch: 4 [288/539]\tLoss: 0.155585\n", "Epoch: 4 [320/539]\tLoss: 0.001898\n", "Epoch: 4 [352/539]\tLoss: 0.026873\n", "Epoch: 4 [384/539]\tLoss: 0.001487\n", "Epoch: 4 [416/539]\tLoss: 0.005737\n", "Epoch: 4 [448/539]\tLoss: 0.001628\n", "Epoch: 4 [480/539]\tLoss: 0.012242\n", "Epoch: 4 [512/539]\tLoss: 0.002932\n", "Epoch 4 - Val F1: 0.9415, Val Accuracy: 0.9415\n", "Novo melhor modelo salvo.\n", "Desempenho no conjunto de teste desta dobra:\n", " precision recall f1-score support\n", "\n", " 0 0.93 0.97 0.95 627\n", " 1 0.96 0.92 0.94 570\n", "\n", " accuracy 0.95 1197\n", " macro avg 0.95 0.94 0.95 1197\n", "weighted avg 0.95 0.95 0.95 1197\n", "\n", "F1 (teste): 0.9456, Accuracy (teste): 0.9457\n", "\n", "=== Fold 3/5 ===\n", "Epoch: 0 [32/539]\tLoss: 0.735917\n", "Epoch: 0 [64/539]\tLoss: 0.592529\n", "Epoch: 0 [96/539]\tLoss: 0.643685\n", "Epoch: 0 [128/539]\tLoss: 0.627175\n", "Epoch: 0 [160/539]\tLoss: 0.575533\n", "Epoch: 0 [192/539]\tLoss: 0.729482\n", "Epoch: 0 [224/539]\tLoss: 0.509104\n", "Epoch: 0 [256/539]\tLoss: 0.786189\n", "Epoch: 0 [288/539]\tLoss: 0.806076\n", "Epoch: 0 [320/539]\tLoss: 0.607852\n", "Epoch: 0 [352/539]\tLoss: 0.473597\n", "Epoch: 0 [384/539]\tLoss: 0.373717\n", "Epoch: 0 [416/539]\tLoss: 0.531733\n", "Epoch: 0 [448/539]\tLoss: 0.715181\n", "Epoch: 0 [480/539]\tLoss: 0.494880\n", "Epoch: 0 [512/539]\tLoss: 0.229518\n", "Epoch 0 - Val F1: 0.7972, Val Accuracy: 0.7975\n", "Novo melhor modelo salvo.\n", "Epoch: 1 [32/539]\tLoss: 0.107594\n", "Epoch: 1 [64/539]\tLoss: 0.467723\n", "Epoch: 1 [96/539]\tLoss: 0.631812\n", "Epoch: 1 [128/539]\tLoss: 0.572087\n", "Epoch: 1 [160/539]\tLoss: 0.248338\n", "Epoch: 1 [192/539]\tLoss: 0.256459\n", "Epoch: 1 [224/539]\tLoss: 0.099010\n", "Epoch: 1 [256/539]\tLoss: 0.167257\n", "Epoch: 1 [288/539]\tLoss: 0.507385\n", "Epoch: 1 [320/539]\tLoss: 0.845850\n", "Epoch: 1 [352/539]\tLoss: 0.117461\n", "Epoch: 1 [384/539]\tLoss: 0.114881\n", "Epoch: 1 [416/539]\tLoss: 0.073258\n", "Epoch: 1 [448/539]\tLoss: 0.160434\n", "Epoch: 1 [480/539]\tLoss: 0.303744\n", "Epoch: 1 [512/539]\tLoss: 0.108967\n", "Epoch 1 - Val F1: 0.9061, Val Accuracy: 0.9061\n", "Novo melhor modelo salvo.\n", "Epoch: 2 [32/539]\tLoss: 0.026718\n", "Epoch: 2 [64/539]\tLoss: 0.107227\n", "Epoch: 2 [96/539]\tLoss: 0.018087\n", "Epoch: 2 [128/539]\tLoss: 0.653934\n", "Epoch: 2 [160/539]\tLoss: 0.084223\n", "Epoch: 2 [192/539]\tLoss: 0.127844\n", "Epoch: 2 [224/539]\tLoss: 0.085503\n", "Epoch: 2 [256/539]\tLoss: 0.012499\n", "Epoch: 2 [288/539]\tLoss: 0.081436\n", "Epoch: 2 [320/539]\tLoss: 0.062079\n", "Epoch: 2 [352/539]\tLoss: 0.023066\n", "Epoch: 2 [384/539]\tLoss: 0.033954\n", "Epoch: 2 [416/539]\tLoss: 0.016665\n", "Epoch: 2 [448/539]\tLoss: 0.035527\n", "Epoch: 2 [480/539]\tLoss: 0.024941\n", "Epoch: 2 [512/539]\tLoss: 0.386871\n", "Epoch 2 - Val F1: 0.9121, Val Accuracy: 0.9123\n", "Novo melhor modelo salvo.\n", "Epoch: 3 [32/539]\tLoss: 0.163918\n", "Epoch: 3 [64/539]\tLoss: 0.009033\n", "Epoch: 3 [96/539]\tLoss: 0.085215\n", "Epoch: 3 [128/539]\tLoss: 0.006444\n", "Epoch: 3 [160/539]\tLoss: 0.025524\n", "Epoch: 3 [192/539]\tLoss: 0.069963\n", "Epoch: 3 [224/539]\tLoss: 0.046311\n", "Epoch: 3 [256/539]\tLoss: 0.039229\n", "Epoch: 3 [288/539]\tLoss: 0.323314\n", "Epoch: 3 [320/539]\tLoss: 0.010894\n", "Epoch: 3 [352/539]\tLoss: 0.042332\n", "Epoch: 3 [384/539]\tLoss: 0.009699\n", "Epoch: 3 [416/539]\tLoss: 0.006716\n", "Epoch: 3 [448/539]\tLoss: 0.012770\n", "Epoch: 3 [480/539]\tLoss: 0.011455\n", "Epoch: 3 [512/539]\tLoss: 0.009429\n", "Epoch 3 - Val F1: 0.9249, Val Accuracy: 0.9248\n", "Novo melhor modelo salvo.\n", "Epoch: 4 [32/539]\tLoss: 0.113521\n", "Epoch: 4 [64/539]\tLoss: 0.011155\n", "Epoch: 4 [96/539]\tLoss: 0.014652\n", "Epoch: 4 [128/539]\tLoss: 0.069262\n", "Epoch: 4 [160/539]\tLoss: 0.006706\n", "Epoch: 4 [192/539]\tLoss: 0.324895\n", "Epoch: 4 [224/539]\tLoss: 0.074128\n", "Epoch: 4 [256/539]\tLoss: 0.006551\n", "Epoch: 4 [288/539]\tLoss: 0.006611\n", "Epoch: 4 [320/539]\tLoss: 0.003768\n", "Epoch: 4 [352/539]\tLoss: 0.005372\n", "Epoch: 4 [384/539]\tLoss: 0.004562\n", "Epoch: 4 [416/539]\tLoss: 0.006518\n", "Epoch: 4 [448/539]\tLoss: 0.005809\n", "Epoch: 4 [480/539]\tLoss: 0.005891\n", "Epoch: 4 [512/539]\tLoss: 0.009314\n", "Epoch 4 - Val F1: 0.9353, Val Accuracy: 0.9353\n", "Novo melhor modelo salvo.\n", "Desempenho no conjunto de teste desta dobra:\n", " precision recall f1-score support\n", "\n", " 0 0.94 0.95 0.94 627\n", " 1 0.94 0.93 0.94 570\n", "\n", " accuracy 0.94 1197\n", " macro avg 0.94 0.94 0.94 1197\n", "weighted avg 0.94 0.94 0.94 1197\n", "\n", "F1 (teste): 0.9407, Accuracy (teste): 0.9407\n", "\n", "=== Fold 4/5 ===\n", "Epoch: 0 [32/539]\tLoss: 0.745997\n", "Epoch: 0 [64/539]\tLoss: 0.682468\n", "Epoch: 0 [96/539]\tLoss: 0.702846\n", "Epoch: 0 [128/539]\tLoss: 0.715030\n", "Epoch: 0 [160/539]\tLoss: 0.769712\n", "Epoch: 0 [192/539]\tLoss: 0.701377\n", "Epoch: 0 [224/539]\tLoss: 0.572981\n", "Epoch: 0 [256/539]\tLoss: 0.526011\n", "Epoch: 0 [288/539]\tLoss: 0.628082\n", "Epoch: 0 [320/539]\tLoss: 0.659288\n", "Epoch: 0 [352/539]\tLoss: 0.354942\n", "Epoch: 0 [384/539]\tLoss: 0.456815\n", "Epoch: 0 [416/539]\tLoss: 0.537916\n", "Epoch: 0 [448/539]\tLoss: 0.372000\n", "Epoch: 0 [480/539]\tLoss: 0.273390\n", "Epoch: 0 [512/539]\tLoss: 0.462089\n", "Epoch 0 - Val F1: 0.8598, Val Accuracy: 0.8601\n", "Novo melhor modelo salvo.\n", "Epoch: 1 [32/539]\tLoss: 0.400376\n", "Epoch: 1 [64/539]\tLoss: 0.292799\n", "Epoch: 1 [96/539]\tLoss: 0.097472\n", "Epoch: 1 [128/539]\tLoss: 0.901339\n", "Epoch: 1 [160/539]\tLoss: 0.617738\n", "Epoch: 1 [192/539]\tLoss: 0.115037\n", "Epoch: 1 [224/539]\tLoss: 0.206575\n", "Epoch: 1 [256/539]\tLoss: 0.186537\n", "Epoch: 1 [288/539]\tLoss: 0.674716\n", "Epoch: 1 [320/539]\tLoss: 0.093894\n", "Epoch: 1 [352/539]\tLoss: 0.126228\n", "Epoch: 1 [384/539]\tLoss: 0.104674\n", "Epoch: 1 [416/539]\tLoss: 0.337742\n", "Epoch: 1 [448/539]\tLoss: 0.612781\n", "Epoch: 1 [480/539]\tLoss: 0.306427\n", "Epoch: 1 [512/539]\tLoss: 0.015238\n", "Epoch 1 - Val F1: 0.8899, Val Accuracy: 0.8914\n", "Novo melhor modelo salvo.\n", "Epoch: 2 [32/539]\tLoss: 0.095481\n", "Epoch: 2 [64/539]\tLoss: 0.016074\n", "Epoch: 2 [96/539]\tLoss: 0.058413\n", "Epoch: 2 [128/539]\tLoss: 0.098460\n", "Epoch: 2 [160/539]\tLoss: 0.010817\n", "Epoch: 2 [192/539]\tLoss: 0.217947\n", "Epoch: 2 [224/539]\tLoss: 0.014442\n", "Epoch: 2 [256/539]\tLoss: 0.010201\n", "Epoch: 2 [288/539]\tLoss: 0.296449\n", "Epoch: 2 [320/539]\tLoss: 0.116354\n", "Epoch: 2 [352/539]\tLoss: 0.036927\n", "Epoch: 2 [384/539]\tLoss: 0.028675\n", "Epoch: 2 [416/539]\tLoss: 0.010761\n", "Epoch: 2 [448/539]\tLoss: 0.022918\n", "Epoch: 2 [480/539]\tLoss: 0.034252\n", "Epoch: 2 [512/539]\tLoss: 0.150974\n", "Epoch 2 - Val F1: 0.9436, Val Accuracy: 0.9436\n", "Novo melhor modelo salvo.\n", "Epoch: 3 [32/539]\tLoss: 0.006742\n", "Epoch: 3 [64/539]\tLoss: 0.015323\n", "Epoch: 3 [96/539]\tLoss: 0.033706\n", "Epoch: 3 [128/539]\tLoss: 0.013044\n", "Epoch: 3 [160/539]\tLoss: 0.003634\n", "Epoch: 3 [192/539]\tLoss: 0.035571\n", "Epoch: 3 [224/539]\tLoss: 0.003196\n", "Epoch: 3 [256/539]\tLoss: 0.078975\n", "Epoch: 3 [288/539]\tLoss: 0.011156\n", "Epoch: 3 [320/539]\tLoss: 0.010516\n", "Epoch: 3 [352/539]\tLoss: 0.030506\n", "Epoch: 3 [384/539]\tLoss: 0.025923\n", "Epoch: 3 [416/539]\tLoss: 0.011039\n", "Epoch: 3 [448/539]\tLoss: 0.077258\n", "Epoch: 3 [480/539]\tLoss: 0.042708\n", "Epoch: 3 [512/539]\tLoss: 0.003598\n", "Epoch 3 - Val F1: 0.9519, Val Accuracy: 0.9520\n", "Novo melhor modelo salvo.\n", "Epoch: 4 [32/539]\tLoss: 0.001914\n", "Epoch: 4 [64/539]\tLoss: 0.043992\n", "Epoch: 4 [96/539]\tLoss: 0.006504\n", "Epoch: 4 [128/539]\tLoss: 0.002461\n", "Epoch: 4 [160/539]\tLoss: 0.012732\n", "Epoch: 4 [192/539]\tLoss: 0.002560\n", "Epoch: 4 [224/539]\tLoss: 0.002396\n", "Epoch: 4 [256/539]\tLoss: 0.006392\n", "Epoch: 4 [288/539]\tLoss: 0.015300\n", "Epoch: 4 [320/539]\tLoss: 0.095689\n", "Epoch: 4 [352/539]\tLoss: 0.007642\n", "Epoch: 4 [384/539]\tLoss: 0.007563\n", "Epoch: 4 [416/539]\tLoss: 0.004592\n", "Epoch: 4 [448/539]\tLoss: 0.035378\n", "Epoch: 4 [480/539]\tLoss: 0.008071\n", "Epoch: 4 [512/539]\tLoss: 0.004615\n", "Epoch 4 - Val F1: 0.9415, Val Accuracy: 0.9415\n", "Desempenho no conjunto de teste desta dobra:\n", " precision recall f1-score support\n", "\n", " 0 0.92 0.94 0.93 628\n", " 1 0.93 0.92 0.92 569\n", "\n", " accuracy 0.93 1197\n", " macro avg 0.93 0.93 0.93 1197\n", "weighted avg 0.93 0.93 0.93 1197\n", "\n", "F1 (teste): 0.9273, Accuracy (teste): 0.9273\n", "\n", "=== Fold 5/5 ===\n", "Epoch: 0 [32/539]\tLoss: 0.736348\n", "Epoch: 0 [64/539]\tLoss: 0.757772\n", "Epoch: 0 [96/539]\tLoss: 0.679295\n", "Epoch: 0 [128/539]\tLoss: 0.660582\n", "Epoch: 0 [160/539]\tLoss: 0.802816\n", "Epoch: 0 [192/539]\tLoss: 0.547177\n", "Epoch: 0 [224/539]\tLoss: 0.691502\n", "Epoch: 0 [256/539]\tLoss: 0.685943\n", "Epoch: 0 [288/539]\tLoss: 0.451294\n", "Epoch: 0 [320/539]\tLoss: 0.541730\n", "Epoch: 0 [352/539]\tLoss: 0.150317\n", "Epoch: 0 [384/539]\tLoss: 0.223378\n", "Epoch: 0 [416/539]\tLoss: 0.731548\n", "Epoch: 0 [448/539]\tLoss: 0.226719\n", "Epoch: 0 [480/539]\tLoss: 0.282167\n", "Epoch: 0 [512/539]\tLoss: 0.424890\n", "Epoch 0 - Val F1: 0.7837, Val Accuracy: 0.7891\n", "Novo melhor modelo salvo.\n", "Epoch: 1 [32/539]\tLoss: 0.398466\n", "Epoch: 1 [64/539]\tLoss: 0.383012\n", "Epoch: 1 [96/539]\tLoss: 0.431615\n", "Epoch: 1 [128/539]\tLoss: 0.117072\n", "Epoch: 1 [160/539]\tLoss: 0.245383\n", "Epoch: 1 [192/539]\tLoss: 0.419279\n", "Epoch: 1 [224/539]\tLoss: 0.275579\n", "Epoch: 1 [256/539]\tLoss: 0.336219\n", "Epoch: 1 [288/539]\tLoss: 0.250066\n", "Epoch: 1 [320/539]\tLoss: 0.293893\n", "Epoch: 1 [352/539]\tLoss: 0.088111\n", "Epoch: 1 [384/539]\tLoss: 0.242597\n", "Epoch: 1 [416/539]\tLoss: 0.061158\n", "Epoch: 1 [448/539]\tLoss: 0.260205\n", "Epoch: 1 [480/539]\tLoss: 0.208200\n", "Epoch: 1 [512/539]\tLoss: 0.113565\n", "Epoch 1 - Val F1: 0.8873, Val Accuracy: 0.8873\n", "Novo melhor modelo salvo.\n", "Epoch: 2 [32/539]\tLoss: 0.070310\n", "Epoch: 2 [64/539]\tLoss: 0.163662\n", "Epoch: 2 [96/539]\tLoss: 0.108127\n", "Epoch: 2 [128/539]\tLoss: 0.281810\n", "Epoch: 2 [160/539]\tLoss: 0.364615\n", "Epoch: 2 [192/539]\tLoss: 0.147060\n", "Epoch: 2 [224/539]\tLoss: 0.062546\n", "Epoch: 2 [256/539]\tLoss: 0.125024\n", "Epoch: 2 [288/539]\tLoss: 0.048165\n", "Epoch: 2 [320/539]\tLoss: 0.241866\n", "Epoch: 2 [352/539]\tLoss: 0.244164\n", "Epoch: 2 [384/539]\tLoss: 0.155161\n", "Epoch: 2 [416/539]\tLoss: 0.300804\n", "Epoch: 2 [448/539]\tLoss: 0.125214\n", "Epoch: 2 [480/539]\tLoss: 0.029340\n", "Epoch: 2 [512/539]\tLoss: 0.132973\n", "Epoch 2 - Val F1: 0.9186, Val Accuracy: 0.9186\n", "Novo melhor modelo salvo.\n", "Epoch: 3 [32/539]\tLoss: 0.182894\n", "Epoch: 3 [64/539]\tLoss: 0.080429\n", "Epoch: 3 [96/539]\tLoss: 0.168509\n", "Epoch: 3 [128/539]\tLoss: 0.070553\n", "Epoch: 3 [160/539]\tLoss: 0.051917\n", "Epoch: 3 [192/539]\tLoss: 0.011117\n", "Epoch: 3 [224/539]\tLoss: 0.080476\n", "Epoch: 3 [256/539]\tLoss: 0.004516\n", "Epoch: 3 [288/539]\tLoss: 0.007997\n", "Epoch: 3 [320/539]\tLoss: 0.106819\n", "Epoch: 3 [352/539]\tLoss: 0.036051\n", "Epoch: 3 [384/539]\tLoss: 0.056000\n", "Epoch: 3 [416/539]\tLoss: 0.017140\n", "Epoch: 3 [448/539]\tLoss: 0.019664\n", "Epoch: 3 [480/539]\tLoss: 0.002534\n", "Epoch: 3 [512/539]\tLoss: 0.016210\n", "Epoch 3 - Val F1: 0.9144, Val Accuracy: 0.9144\n", "Epoch: 4 [32/539]\tLoss: 0.040328\n", "Epoch: 4 [64/539]\tLoss: 0.005345\n", "Epoch: 4 [96/539]\tLoss: 0.005071\n", "Epoch: 4 [128/539]\tLoss: 0.007578\n", "Epoch: 4 [160/539]\tLoss: 0.019537\n", "Epoch: 4 [192/539]\tLoss: 0.004254\n", "Epoch: 4 [224/539]\tLoss: 0.004656\n", "Epoch: 4 [256/539]\tLoss: 0.028694\n", "Epoch: 4 [288/539]\tLoss: 0.009497\n", "Epoch: 4 [320/539]\tLoss: 0.006693\n", "Epoch: 4 [352/539]\tLoss: 0.021709\n", "Epoch: 4 [384/539]\tLoss: 0.007690\n", "Epoch: 4 [416/539]\tLoss: 0.003045\n", "Epoch: 4 [448/539]\tLoss: 0.019487\n", "Epoch: 4 [480/539]\tLoss: 0.002427\n", "Epoch: 4 [512/539]\tLoss: 0.009079\n", "Epoch 4 - Val F1: 0.9270, Val Accuracy: 0.9269\n", "Novo melhor modelo salvo.\n", "Desempenho no conjunto de teste desta dobra:\n", " precision recall f1-score support\n", "\n", " 0 0.92 0.94 0.93 627\n", " 1 0.93 0.91 0.92 569\n", "\n", " accuracy 0.92 1196\n", " macro avg 0.93 0.92 0.92 1196\n", "weighted avg 0.92 0.92 0.92 1196\n", "\n", "F1 (teste): 0.9247, Accuracy (teste): 0.9247\n", "\n", "=== Resultados Médios da Validação Cruzada ===\n", "F1 médio: 0.9334 (+/- 0.0082)\n", "Acurácia média: 0.9335 (+/- 0.0082)\n" ] } ], "source": [ "# Modelo\n", "class CustomBERTModel(nn.Module):\n", " def __init__(self, model_name, nclasses, unfreeze_layers):\n", " super(CustomBERTModel, self).__init__()\n", " self.bert = AutoModel.from_pretrained(model_name)\n", " self.dropout = nn.Dropout(0.3)\n", " self.classifier = nn.Linear(self.bert.config.hidden_size, nclasses)\n", "\n", " # Congelar tudo inicialmente\n", " for param in self.bert.parameters():\n", " param.requires_grad = False\n", "\n", " # Descongelar as últimas unfreeze_layers camadas\n", " if unfreeze_layers > 0:\n", " for param in self.bert.encoder.layer[-unfreeze_layers:].parameters():\n", " param.requires_grad = True\n", "\n", " def forward(self, input_ids, attention_mask, token_type_ids=None):\n", " outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)\n", " pooled_output = outputs.pooler_output\n", " dropped_out = self.dropout(pooled_output)\n", " logits = self.classifier(dropped_out)\n", " return logits\n", "\n", "def evaluate(model, dataloader):\n", " model.eval()\n", " y_real, y_pred = [], []\n", " with torch.no_grad():\n", " for inputs, labels in dataloader:\n", " inputs = {key: val.to(device) for key, val in inputs.items()}\n", " labels = labels.to(device)\n", " logits = model(**inputs)\n", " pred_labels = torch.argmax(logits, 1)\n", "\n", " y_real.extend(labels.cpu().tolist())\n", " y_pred.extend(pred_labels.cpu().tolist())\n", "\n", " f1 = f1_score(y_real, y_pred, average='weighted')\n", " acc = accuracy_score(y_real, y_pred)\n", " return f1, acc, (y_real, y_pred)\n", "\n", "\n", "# Cross-validation\n", "k = 5\n", "skf = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)\n", "X = data.index.values\n", "y = data['contra'].values\n", "\n", "f1_scores = []\n", "acc_scores = []\n", "\n", "tokenizer = AutoTokenizer.from_pretrained(model_name, do_lower_case=False)\n", "\n", "fold_num = 1\n", "for train_val_idx, test_idx in skf.split(X, y):\n", " print(f\"\\n=== Fold {fold_num}/{k} ===\")\n", "\n", " # Separamos test fold\n", " test_data = data.iloc[test_idx]\n", "\n", " # A partir do train_val_idx, dividimos em train e val\n", " train_val_data = data.iloc[train_val_idx]\n", " train_data, val_data = train_test_split(train_val_data, \n", " test_size=0.1, \n", " random_state=seed, \n", " stratify=train_val_data['contra'])\n", " \n", " # Criar datasets e dataloaders\n", " train_dataset = CustomDataset(train_data, tokenizer, max_length)\n", " val_dataset = CustomDataset(val_data, tokenizer, max_length)\n", " test_dataset = CustomDataset(test_data, tokenizer, max_length)\n", "\n", " traindata = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n", " valdata = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n", " testdata = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n", "\n", " model = CustomBERTModel(model_name, nclasses, unfreeze_layers).to(device)\n", " optimizer = optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=learning_rate)\n", "\n", " loss_fn = nn.CrossEntropyLoss()\n", "\n", " total_steps = len(traindata) * nepochs\n", " scheduler = get_linear_schedule_with_warmup(optimizer, \n", " num_warmup_steps=int(0.1 * total_steps), \n", " num_training_steps=total_steps)\n", "\n", " max_f1, repeat = 0, 0\n", " best_model_path = os.path.join(write_path, f'best_model_fold{fold_num}.pth')\n", "\n", " for epoch in range(nepochs):\n", " model.train()\n", " losses = []\n", " for batch_idx, (inputs, labels) in enumerate(traindata):\n", " inputs = {key: val.to(device) for key, val in inputs.items()}\n", " labels = labels.to(device)\n", "\n", " logits = model(**inputs)\n", " loss = loss_fn(logits, labels)\n", " losses.append(float(loss))\n", "\n", " # Backprop\n", " loss.backward()\n", " optimizer.step()\n", " scheduler.step()\n", " optimizer.zero_grad()\n", "\n", " if (batch_idx + 1) % batch_status == 0:\n", " print(f'Epoch: {epoch} [{batch_idx + 1}/{len(traindata)}]\\tLoss: {loss:.6f}')\n", "\n", " f1_val, acc_val, _ = evaluate(model, valdata)\n", " print(f'Epoch {epoch} - Val F1: {f1_val:.4f}, Val Accuracy: {acc_val:.4f}')\n", "\n", " if f1_val > max_f1:\n", " torch.save(model.state_dict(), best_model_path)\n", " max_f1 = f1_val\n", " repeat = 0\n", " print('Novo melhor modelo salvo.')\n", " else:\n", " repeat += 1\n", "\n", " if repeat == early_stop:\n", " print('Early stopping atingido.')\n", " break\n", "\n", " # Avaliar no teste\n", " state_dict = torch.load(best_model_path, weights_only=True)\n", " model.load_state_dict(state_dict)\n", " f1_test, acc_test, (y_real, y_pred) = evaluate(model, testdata)\n", "\n", " print(\"Desempenho no conjunto de teste desta dobra:\")\n", " print(classification_report(y_real, y_pred, target_names=['0', '1']))\n", " print(f\"F1 (teste): {f1_test:.4f}, Accuracy (teste): {acc_test:.4f}\")\n", "\n", " f1_scores.append(f1_test)\n", " acc_scores.append(acc_test)\n", "\n", " fold_num += 1\n", "\n", "# Resultados médios da validação cruzada\n", "print(\"\\n=== Resultados Médios da Validação Cruzada ===\")\n", "print(f\"F1 médio: {statistics.mean(f1_scores):.4f} (+/- {statistics.pstdev(f1_scores):.4f})\")\n", "print(f\"Acurácia média: {statistics.mean(acc_scores):.4f} (+/- {statistics.pstdev(acc_scores):.4f})\")\n" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.7" } }, "nbformat": 4, "nbformat_minor": 5 }