{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "0c24ca36-1782-4ce8-8094-6f6528dada19", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 302, "referenced_widgets": [ "ffbb5c6a3b7f41a2a20769204dde9281", "1f309cae286e4bbebc79393e43b32205", "a66cb4bca4fb45c893549fc9bace8150", "9f626afdd1ef42adb413b98088ef49da", "9135fa47675a417eac79514c9c071c1b", "73a82d5ee0b14f4599904fda0aff3d67", "5b9b62cc01444ce98148697c41a05c90", "2518a6ae25b7427faa155115efa4d921", "584e7cacfe634c23b9902c4e655219bd", "d672ac2ea2814aa18b41501e6a0aee25", "d902c2ff97e240b4ae6783d8824e3822", "966c62fdbf0843f498e2aa226e9e761c", "7e50b448b5544a8cb6a38250623c0331", "30b61336ce554e4284bfc64f7fbbbae8", "1385025409a74dfda9c3d2c9c75b914a", "382ca19ee53a42b2918c6cf9188fa1ca", "42a33944551b46769ac206e4f15bc77e" ] }, "id": "0c24ca36-1782-4ce8-8094-6f6528dada19", "outputId": "f7bf40c8-0e7c-4dfe-cf53-990ff36b13c8" }, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "VBox(children=(HTML(value='
=2.6.3\"\n", "==((====))== Unsloth 2025.1.7: Fast Gemma2 patching. Transformers: 4.47.1.\n", " \\\\ /| GPU: NVIDIA A100-SXM4-40GB. Max memory: 39.564 GB. Platform: Linux.\n", "O^O/ \\_/ \\ Torch: 2.5.1+cu121. CUDA: 8.0. CUDA Toolkit: 12.1. Triton: 3.1.0\n", "\\ / Bfloat16 = TRUE. FA [Xformers = 0.0.29.post1. FA2 = False]\n", " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "Loading checkpoint shards: 0%| | 0/4 [00:00 0 ! Suggested 8, 16, 32, 64, 128\n", " target_modules = [\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n", " \"gate_proj\", \"up_proj\", \"down_proj\",],\n", " lora_alpha = 16,\n", " lora_dropout = 0, # Supports any, but = 0 is optimized\n", " bias = \"none\", # Supports any, but = \"none\" is optimized\n", " # [NEW] \"unsloth\" uses 30% less VRAM, fits 2x larger batch sizes!\n", " use_gradient_checkpointing = \"unsloth\", # True or \"unsloth\" for very long context\n", " random_state = 3407,\n", " use_rslora = False, # We support rank stabilized LoRA\n", " loftq_config = None, # And LoftQ\n", ")" ] }, { "cell_type": "code", "execution_count": 5, "id": "35603404-a7a0-4f24-92de-fd778500df99", "metadata": { "id": "35603404-a7a0-4f24-92de-fd778500df99", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "d6b276f4-a571-4703-dc33-efe7b355263c" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Requirement already satisfied: datasets in /usr/local/lib/python3.11/dist-packages (3.2.0)\n", "Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (4.67.1)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from datasets) (3.17.0)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/dist-packages (from datasets) (1.26.4)\n", "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.11/dist-packages (from datasets) (17.0.0)\n", "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.11/dist-packages (from datasets) (0.3.8)\n", "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (from datasets) (2.2.2)\n", "Requirement already satisfied: requests>=2.32.2 in /usr/local/lib/python3.11/dist-packages (from datasets) (2.32.3)\n", "Requirement already satisfied: xxhash in /usr/local/lib/python3.11/dist-packages (from datasets) (3.5.0)\n", "Requirement already satisfied: multiprocess<0.70.17 in /usr/local/lib/python3.11/dist-packages (from datasets) (0.70.16)\n", "Requirement already satisfied: fsspec<=2024.9.0,>=2023.1.0 in /usr/local/lib/python3.11/dist-packages (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets) (2024.9.0)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/dist-packages (from datasets) (3.11.11)\n", "Requirement already satisfied: huggingface-hub>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from datasets) (0.27.1)\n", "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from datasets) (24.2)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/dist-packages (from datasets) (6.0.2)\n", "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (2.4.4)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (1.3.2)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (24.3.0)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (1.5.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (6.1.0)\n", "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (0.2.1)\n", "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (1.18.3)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub>=0.23.0->datasets) (4.12.2)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets) (3.4.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets) (3.10)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets) (2.3.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets) (2024.12.14)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets) (2.8.2)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets) (2024.2)\n", "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets) (2025.1)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.17.0)\n" ] } ], "source": [ "!pip install datasets tqdm\n", "import pandas as pd\n", "from datasets import load_dataset\n", "import os" ] }, { "cell_type": "code", "execution_count": 6, "id": "2c49fafe-1e9a-4e29-8d85-94ff79f8c9c0", "metadata": { "id": "2c49fafe-1e9a-4e29-8d85-94ff79f8c9c0" }, "outputs": [], "source": [ "FastLanguageModel.for_inference(model)\n", "from tqdm import tqdm\n", "tqdm.pandas()" ] }, { "cell_type": "code", "execution_count": 7, "id": "1749c745-d1fb-430b-9469-4913bb2a6cb5", "metadata": { "id": "1749c745-d1fb-430b-9469-4913bb2a6cb5", "colab": { "base_uri": "https://localhost:8080/", "height": 441 }, "outputId": "14026901-cd3e-4f00-bf64-0c4958296272" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "14042\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [ " Input Output \\\n", "0 Q पर दिए गए क्षेत्र विस्तार Q(sqrt(2), sqrt(3)... B \n", "1 मान लीजिए S_5 में p = (1, 2, 5, 4)(2, 3) है. S... C \n", "2 दिए गए बहुपद के संकेतित परिमित क्षेत्र में गुण... D \n", "3 कथन 1 | नॉन-एबेलियन ग्रुप का एक फ़ैक्टर ग्रुप ... B \n", "4 दिए गए बहुपद रिंग में दिए गए बहुपदों का गुणनफल... B \n", "... ... ... \n", "14037 चीन और कोरिया में धार्मिक परम्पराओं का मुख्य क... A \n", "14038 हान राजवंश के समय सूखे के दौरान आम लोग किससे अ... C \n", "14039 धर्मशास्त्रीय शब्द होमोओसिओस का अर्थ निम्नलिखि... B \n", "14040 जापानी मूल मिथक के अनुसार, अमातरासु को किसने अ... B \n", "14041 ऑगस्टस का नुमेन निम्नलिखित में से किस विशेषता ... A \n", "\n", " Sub-Domain Dataset \n", "0 abstract_algebra MMLU \n", "1 abstract_algebra MMLU \n", "2 abstract_algebra MMLU \n", "3 abstract_algebra MMLU \n", "4 abstract_algebra MMLU \n", "... ... ... \n", "14037 world_religions MMLU \n", "14038 world_religions MMLU \n", "14039 world_religions MMLU \n", "14040 world_religions MMLU \n", "14041 world_religions MMLU \n", "\n", "[14042 rows x 4 columns]" ], "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
InputOutputSub-DomainDataset
0Q पर दिए गए क्षेत्र विस्तार Q(sqrt(2), sqrt(3)...Babstract_algebraMMLU
1मान लीजिए S_5 में p = (1, 2, 5, 4)(2, 3) है. S...Cabstract_algebraMMLU
2दिए गए बहुपद के संकेतित परिमित क्षेत्र में गुण...Dabstract_algebraMMLU
3कथन 1 | नॉन-एबेलियन ग्रुप का एक फ़ैक्टर ग्रुप ...Babstract_algebraMMLU
4दिए गए बहुपद रिंग में दिए गए बहुपदों का गुणनफल...Babstract_algebraMMLU
...............
14037चीन और कोरिया में धार्मिक परम्पराओं का मुख्य क...Aworld_religionsMMLU
14038हान राजवंश के समय सूखे के दौरान आम लोग किससे अ...Cworld_religionsMMLU
14039धर्मशास्त्रीय शब्द होमोओसिओस का अर्थ निम्नलिखि...Bworld_religionsMMLU
14040जापानी मूल मिथक के अनुसार, अमातरासु को किसने अ...Bworld_religionsMMLU
14041ऑगस्टस का नुमेन निम्नलिखित में से किस विशेषता ...Aworld_religionsMMLU
\n", "

14042 rows × 4 columns

\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", " \n", "\n", "\n", "\n", " \n", "
\n", "\n", "
\n", " \n", " \n", " \n", "
\n", "\n", "
\n", "
\n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "variable_name": "df", "summary": "{\n \"name\": \"df\",\n \"rows\": 14042,\n \"fields\": [\n {\n \"column\": \"Input\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 13981,\n \"samples\": [\n \"\\u092c\\u0941\\u0928\\u094d\\u092f\\u093e\\u0935\\u093e\\u092f\\u0930\\u0938 \\u092a\\u0930\\u093f\\u0935\\u093e\\u0930 \\u0915\\u0940 \\u090f\\u0915 \\u092a\\u094d\\u0930\\u092e\\u0941\\u0916 \\u0935\\u093f\\u0936\\u0947\\u0937\\u0924\\u093e \\u0915\\u094c\\u0928 \\u0938\\u0940 \\u0939\\u0948? ### A) \\u092c\\u094c\\u0928\\u094b\\u0902, \\u092e\\u091a\\u094d\\u091b\\u0930\\u094b\\u0902, \\u0930\\u0947\\u0924 \\u0915\\u0940 \\u092e\\u0915\\u094d\\u0916\\u093f\\u092f\\u094b\\u0902, \\u0915\\u0941\\u091f\\u0915\\u0940 \\u0915\\u0947 \\u0932\\u0917\\u093e\\u0924\\u093e\\u0930 \\u0938\\u0902\\u0915\\u094d\\u0930\\u092e\\u0923 \\u0938\\u0947 \\u092c\\u0922\\u093c\\u0928\\u093e B) \\u092e\\u0941\\u0916\\u094d\\u092f \\u0930\\u0942\\u092a \\u0938\\u0947 \\u092e\\u091a\\u094d\\u091b\\u0930\\u094b\\u0902 \\u0938\\u0947 \\u092b\\u0948\\u0932\\u0928\\u093e C) \\u0906\\u0928\\u0941\\u0935\\u0902\\u0936\\u093f\\u0915 \\u092a\\u0941\\u0928\\u0930\\u094d\\u0938\\u0902\\u092f\\u094b\\u091c\\u0928 \\u092f\\u093e \\u092a\\u0941\\u0928\\u0930\\u094d\\u0935\\u093f\\u0928\\u094d\\u092f\\u093e\\u0938 D) \\u091a\\u0942\\u0939\\u094b\\u0902 \\u0915\\u0947 \\u0915\\u093e\\u091f\\u0928\\u0947 \\u0938\\u0947 \\u092e\\u0928\\u0941\\u0937\\u094d\\u092f\\u094b\\u0902 \\u092e\\u0947\\u0902 \\u092b\\u0948\\u0932\\u093e\\u0935 ### MCQ ### \",\n \"70 \\u0935\\u0930\\u094d\\u0937\\u0940\\u092f \\u090f\\u0915 \\u092e\\u0939\\u093f\\u0932\\u093e \\u0905\\u092a\\u0928\\u0947 \\u0939\\u093e\\u0925\\u094b\\u0902 \\u0915\\u0947 \\u0915\\u0902\\u092a\\u0928 \\u0915\\u0940 \\u0938\\u092e\\u0938\\u094d\\u092f\\u093e \\u0915\\u0947 \\u0915\\u093e\\u0930\\u0923 \\u0915\\u093e\\u0930\\u094d\\u092f\\u093e\\u0932\\u092f \\u0906\\u0924\\u0940 \\u0939\\u0948\\u0964 \\u092f\\u0939 \\u0915\\u0902\\u092a\\u0928 \\u0909\\u0938\\u0915\\u0947 \\u091c\\u0940\\u0935\\u0928 \\u0915\\u0947 \\u0905\\u0927\\u093f\\u0915\\u093e\\u0902\\u0936 \\u0938\\u092e\\u092f \\u0938\\u0947 \\u092e\\u094c\\u091c\\u0942\\u0926 \\u0939\\u0948 \\u0914\\u0930 \\u0936\\u0941\\u0930\\u0942 \\u092e\\u0947\\u0902 \\u0939\\u0932\\u094d\\u0915\\u093e \\u0925\\u093e \\u0914\\u0930 \\u0915\\u0947\\u0935\\u0932 \\u0924\\u092d\\u0940 \\u0939\\u094b\\u0924\\u093e \\u0925\\u093e \\u091c\\u092c \\u0935\\u0939 \\u0925\\u0915\\u0940 \\u0939\\u0941\\u0908 \\u092f\\u093e \\u0924\\u0928\\u093e\\u0935\\u0917\\u094d\\u0930\\u0938\\u094d\\u0924 \\u0939\\u094b\\u0924\\u0940 \\u0925\\u0940\\u0964 \\u092a\\u093f\\u091b\\u0932\\u0947 \\u092e\\u0939\\u0940\\u0928\\u0947 \\u0915\\u0947 \\u0926\\u094c\\u0930\\u093e\\u0928 \\u0915\\u0902\\u092a\\u0928 \\u0905\\u0927\\u093f\\u0915 \\u0927\\u094d\\u092f\\u093e\\u0928 \\u0926\\u0947\\u0928\\u0947 \\u092f\\u094b\\u0917\\u094d\\u092f \\u0914\\u0930 \\u0932\\u0917\\u093e\\u0924\\u093e\\u0930 \\u0939\\u094b \\u0917\\u092f\\u093e \\u0939\\u0948\\u0964 \\u0915\\u0902\\u092a\\u0928 \\u0915\\u0940 \\u0938\\u092e\\u0938\\u094d\\u092f\\u093e \\u0915\\u0947 \\u0915\\u093e\\u0930\\u0923 \\u0905\\u092c \\u0935\\u0939 \\u0905\\u0928\\u094d\\u092f \\u0932\\u094b\\u0917\\u094b\\u0902 \\u0915\\u0947 \\u0938\\u093e\\u0925 \\u0916\\u093e\\u0928\\u0947 \\u092e\\u0947\\u0902 \\u0936\\u0930\\u094d\\u092e \\u092e\\u0939\\u0938\\u0942\\u0938 \\u0915\\u0930\\u0924\\u0940 \\u0939\\u0948\\u0964\\u092e\\u0930\\u0940\\u091c \\u092a\\u093f\\u091b\\u0932\\u0947 3 \\u0938\\u092a\\u094d\\u0924\\u093e\\u0939 \\u0938\\u0947 \\u092b\\u094d\\u0932\\u0941\\u0913\\u0915\\u094d\\u0938\\u0947\\u091f\\u0940\\u0928 \\u0932\\u0947 \\u0930\\u0939\\u0940 \\u0939\\u0948 \\u0924\\u093e\\u0915\\u093f 2 \\u092e\\u0939\\u0940\\u0928\\u0947 \\u092a\\u0939\\u0932\\u0947 \\u0909\\u0938\\u0915\\u0947 \\u092a\\u0924\\u093f \\u0915\\u0940 \\u092e\\u0943\\u0924\\u094d\\u092f\\u0941 \\u0938\\u0947 \\u0909\\u092c\\u0930\\u0928\\u0947 \\u092e\\u0947\\u0902 \\u0909\\u0938\\u0947 \\u092e\\u0926\\u0926 \\u092e\\u093f\\u0932 \\u0938\\u0915\\u0947\\u0964 \\u0932\\u093f\\u0938\\u093f\\u0928\\u094b\\u092a\\u094d\\u0930\\u093f\\u0932 \\u0938\\u0947 \\u0928\\u093f\\u092f\\u0902\\u0924\\u094d\\u0930\\u093f\\u0924 \\u0906\\u0935\\u0936\\u094d\\u092f\\u0915 \\u0909\\u091a\\u094d\\u091a \\u0930\\u0915\\u094d\\u0924\\u091a\\u093e\\u092a \\u0914\\u0930 \\u090f\\u091f\\u094b\\u0930\\u0935\\u093e\\u0938\\u094d\\u091f\\u0947\\u091f\\u093f\\u0928 \\u0938\\u0947 \\u0928\\u093f\\u092f\\u0902\\u0924\\u094d\\u0930\\u093f\\u0924 \\u0939\\u093e\\u0907\\u092a\\u0930\\u0932\\u093f\\u092a\\u093f\\u0921\\u093f\\u092e\\u093f\\u092f\\u093e \\u0915\\u0947 \\u0932\\u093f\\u090f \\u091a\\u093f\\u0915\\u093f\\u0924\\u094d\\u0938\\u093e \\u0907\\u0924\\u093f\\u0939\\u093e\\u0938 \\u092d\\u0940 \\u0909\\u0932\\u094d\\u0932\\u0947\\u0916\\u0928\\u0940\\u092f \\u0939\\u0948\\u0964 \\u091c\\u094b\\u0921\\u093c\\u094b\\u0902 \\u0915\\u0947 \\u0926\\u0930\\u094d\\u0926 \\u0915\\u0947 \\u0932\\u093f\\u090f \\u0909\\u0938\\u0915\\u0940 \\u090f\\u0915\\u092e\\u093e\\u0924\\u094d\\u0930 \\u0905\\u0928\\u094d\\u092f \\u0926\\u0935\\u093e \\u0915\\u092d\\u0940-\\u0915\\u092d\\u0940 \\u0907\\u092c\\u0941\\u092a\\u094d\\u0930\\u094b\\u092b\\u0947\\u0928 \\u0939\\u0948\\u0964 \\u0935\\u0939 \\u092e\\u0939\\u0940\\u0928\\u0947 \\u092e\\u0947\\u0902 \\u090f\\u0915 \\u0938\\u0947 \\u0926\\u094b \\u092c\\u093e\\u0930 \\u0936\\u0930\\u093e\\u092c \\u092a\\u0940\\u0924\\u0940 \\u0925\\u0940 \\u0932\\u0947\\u0915\\u093f\\u0928 \\u0905\\u092c \\u0935\\u0939 \\u0930\\u094b\\u091c\\u093e\\u0928\\u093e \\u090f\\u0915 \\u0917\\u093f\\u0932\\u093e\\u0938 \\u0935\\u093e\\u0907\\u0928 \\u092a\\u0940\\u0924\\u0940 \\u0939\\u0948 \\u0915\\u094d\\u092f\\u094b\\u0902\\u0915\\u093f, \\u0909\\u0938\\u0915\\u093e \\u0915\\u0939\\u0928\\u093e \\u0939\\u0948 \\u0915\\u093f \\u0907\\u0938\\u0938\\u0947 \\u0909\\u0938\\u0915\\u093e \\u0915\\u0902\\u092a\\u0928 \\u0915\\u092e \\u0939\\u094b\\u0924\\u093e \\u0939\\u0948\\u0964 \\u0909\\u0938\\u0915\\u0940 \\u0932\\u0902\\u092c\\u093e\\u0908 168 \\u0938\\u0947\\u092e\\u0940 (5 \\u092b\\u0940\\u091f 6 \\u0907\\u0902\\u091a) \\u0939\\u0948 \\u0914\\u0930 \\u0909\\u0938\\u0915\\u093e \\u0935\\u091c\\u0928 70 \\u0915\\u093f\\u0932\\u094b\\u0917\\u094d\\u0930\\u093e\\u092e (155 \\u092a\\u093e\\u0909\\u0902\\u0921) \\u0939\\u0948; \\u092c\\u0940\\u090f\\u092e\\u0906\\u0908 25 \\u0915\\u093f\\u0932\\u094b\\u0917\\u094d\\u0930\\u093e\\u092e/\\u092e\\u09402 \\u0939\\u0948\\u0964 \\u092e\\u0939\\u0924\\u094d\\u0935\\u092a\\u0942\\u0930\\u094d\\u0923 \\u0938\\u0902\\u0915\\u0947\\u0924 \\u0939\\u0948\\u0902 \\u0924\\u093e\\u092a\\u092e\\u093e\\u0928 36.4\\u00b0C (97.6\\u00b0F), \\u0928\\u093e\\u0921\\u093c\\u0940 80/\\u092e\\u093f\\u0928\\u091f, \\u0936\\u094d\\u0935\\u0938\\u0928 18/\\u092e\\u093f\\u0928\\u091f, \\u0924\\u0925\\u093e \\u0930\\u0915\\u094d\\u0924\\u091a\\u093e\\u092a 130/85 mm Hg. \\u0936\\u093e\\u0930\\u0940\\u0930\\u093f\\u0915 \\u092a\\u0930\\u0940\\u0915\\u094d\\u0937\\u0923 \\u0938\\u0947 \\u0926\\u094b\\u0928\\u094b\\u0902 \\u0939\\u093e\\u0925\\u094b\\u0902 \\u092e\\u0947\\u0902 \\u092e\\u0927\\u094d\\u092f\\u092e \\u0915\\u0902\\u092a\\u0928 \\u0926\\u093f\\u0916\\u093e\\u0908 \\u0926\\u0947\\u0924\\u093e \\u0939\\u0948 \\u091c\\u094b \\u0906\\u0930\\u093e\\u092e \\u0915\\u0947 \\u0938\\u092e\\u092f \\u0928\\u0939\\u0940\\u0902 \\u0939\\u094b\\u0924\\u093e\\u0964 \\u092a\\u0942\\u0930\\u094d\\u0923 \\u0930\\u0915\\u094d\\u0924 \\u0917\\u0923\\u0928\\u093e, \\u0938\\u0940\\u0930\\u092e \\u0930\\u0938\\u093e\\u092f\\u0928 \\u092a\\u094d\\u0930\\u094b\\u092b\\u093c\\u093e\\u0907\\u0932, \\u0924\\u0925\\u093e \\u0938\\u0940\\u0930\\u092e \\u0925\\u093e\\u092f\\u0930\\u0949\\u092f\\u0921 \\u092b\\u093c\\u0902\\u0915\\u094d\\u0936\\u0928 \\u092a\\u0930\\u0940\\u0915\\u094d\\u0937\\u0923 \\u0915\\u093e \\u0906\\u0926\\u0947\\u0936 \\u0926\\u093f\\u092f\\u093e \\u0917\\u092f\\u093e \\u0939\\u0948 \\u0924\\u0925\\u093e \\u092a\\u0930\\u093f\\u0923\\u093e\\u092e \\u0932\\u0902\\u092c\\u093f\\u0924 \\u0939\\u0948\\u0902\\u0964 \\u0930\\u094b\\u0917\\u0940 \\u0915\\u0947 \\u0915\\u0902\\u092a\\u0928 \\u0915\\u0947 \\u092c\\u093f\\u0917\\u0921\\u093c\\u0928\\u0947 \\u0915\\u093e \\u0938\\u092c\\u0938\\u0947 \\u0938\\u0902\\u092d\\u093e\\u0935\\u093f\\u0924 \\u0915\\u093e\\u0930\\u0923 \\u0928\\u093f\\u092e\\u094d\\u0928\\u0932\\u093f\\u0916\\u093f\\u0924 \\u092e\\u0947\\u0902 \\u0938\\u0947 \\u0915\\u094c\\u0928 \\u0938\\u093e \\u0939\\u0948? ### A) \\u092b\\u094d\\u0932\\u0941\\u0913\\u0915\\u094d\\u0938\\u0947\\u091f\\u0940\\u0928 \\u0925\\u0947\\u0930\\u0947\\u092a\\u0940 \\u0915\\u093e \\u092a\\u094d\\u0930\\u0924\\u093f\\u0915\\u0942\\u0932 \\u092a\\u094d\\u0930\\u092d\\u093e\\u0935 B) \\u0936\\u094b\\u0915 \\u092a\\u094d\\u0930\\u0924\\u093f\\u0915\\u094d\\u0930\\u093f\\u092f\\u093e C) \\u092a\\u094d\\u0930\\u093e\\u0930\\u0902\\u092d\\u093f\\u0915 \\u092a\\u093e\\u0930\\u094d\\u0915\\u093f\\u0902\\u0938\\u0928 \\u0930\\u094b\\u0917 D) \\u0936\\u0930\\u093e\\u092c \\u0915\\u0940 \\u0916\\u092a\\u0924 \\u092e\\u0947\\u0902 \\u0935\\u0943\\u0926\\u094d\\u0927\\u093f ### MCQ ### \",\n \"\\u0932\\u0917\\u092d\\u0917 \\u0915\\u093f\\u0924\\u0928\\u0947 \\u0936\\u094b\\u0915\\u0917\\u094d\\u0930\\u0938\\u094d\\u0924 \\u0935\\u094d\\u092f\\u0915\\u094d\\u0924\\u093f\\u092f\\u094b\\u0902 \\u0915\\u094b \\u0915\\u0941\\u091b \\u092a\\u0947\\u0936\\u0947\\u0935\\u0930 \\u092e\\u0926\\u0926 \\u0915\\u0940 \\u0906\\u0935\\u0936\\u094d\\u092f\\u0915\\u0924\\u093e \\u0939\\u094b\\u0924\\u0940 \\u0939\\u0948? ### A) 10% \\u0938\\u0947 \\u0915\\u092e B) \\u0932\\u0917\\u092d\\u0917 25% C) \\u0932\\u0917\\u092d\\u0917 \\u0906\\u0927\\u0947 D) 50% \\u0938\\u0947 \\u0905\\u0927\\u093f\\u0915 ### MCQ ### \"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Output\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 4,\n \"samples\": [\n \"C\",\n \"A\",\n \"B\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Sub-Domain\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 57,\n \"samples\": [\n \"abstract_algebra\",\n \"college_biology\",\n \"high_school_us_history\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Dataset\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 1,\n \"samples\": [\n \"MMLU\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" } }, "metadata": {}, "execution_count": 7 } ], "source": [ "dataset = load_dataset(\"1-800-LLMs/Test-Collection\", data_files=\"MMMLU_H.csv\", split=\"train\")\n", "df = dataset.to_pandas()\n", "print(len(df))\n", "df" ] }, { "cell_type": "code", "execution_count": 8, "id": "251f9198-60d4-4ec1-88a8-f344234be2de", "metadata": { "id": "251f9198-60d4-4ec1-88a8-f344234be2de" }, "outputs": [], "source": [ "df['tok'] = df['Input'].apply(lambda x: len(tokenizer.encode(x)))" ] }, { "cell_type": "code", "execution_count": 9, "id": "dbffda0c-e536-4891-8862-e1e5936eb6be", "metadata": { "id": "dbffda0c-e536-4891-8862-e1e5936eb6be", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "0f77afb0-7716-41f7-db8d-6a4a24bfbae6" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Average 'tok' value: 180.577624270047\n", "Max 'tok' value: 1917\n" ] } ], "source": [ "print(f\"Average 'tok' value: {df['tok'].mean()}\")\n", "print(f\"Max 'tok' value: {df['tok'].max()}\")" ] }, { "cell_type": "code", "execution_count": 10, "id": "9409ad7f-697c-4533-8d2f-399928ca9033", "metadata": { "id": "9409ad7f-697c-4533-8d2f-399928ca9033" }, "outputs": [], "source": [ "df = df.sort_values('tok', ascending=False)" ] }, { "cell_type": "code", "execution_count": 11, "id": "26544979-f1f2-4622-be64-9f8559a99460", "metadata": { "id": "26544979-f1f2-4622-be64-9f8559a99460", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "81f4e7e6-9fea-44df-eb5e-ae1c925f610b" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Requirement already satisfied: matplotlib in /usr/local/lib/python3.11/dist-packages (3.10.0)\n", "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.3.1)\n", "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (0.12.1)\n", "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (4.55.5)\n", "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.4.8)\n", "Requirement already satisfied: numpy>=1.23 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.26.4)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (24.2)\n", "Requirement already satisfied: pillow>=8 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (11.1.0)\n", "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (3.2.1)\n", "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (2.8.2)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/dist-packages (from python-dateutil>=2.7->matplotlib) (1.17.0)\n" ] } ], "source": [ "!pip install matplotlib" ] }, { "cell_type": "code", "execution_count": 12, "id": "8ce90df6-271e-460b-97fe-30a2480c78b6", "metadata": { "id": "8ce90df6-271e-460b-97fe-30a2480c78b6", "colab": { "base_uri": "https://localhost:8080/", "height": 564 }, "outputId": "e913ae66-a41a-4632-83db-08c818fa67db" }, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "
" ], "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1sAAAIjCAYAAAD1OgEdAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAfVdJREFUeJzt3X18zfX/x/Hn2cXZhdlmmJnYhqK5JhdLroq5WLrSt0KhRPmRUEIXclERXZdvpb5R3ygp6esizLUYikYukotJyaiYYWyzfX5/+O58HZvtjHN2Pmd73G+3c8v5fN7nc16f8zpmzz6fz/tjMQzDEAAAAADAqbzcXQAAAAAAlEaELQAAAABwAcIWAAAAALgAYQsAAAAAXICwBQAAAAAuQNgCAAAAABcgbAEAAACACxC2AAAAAMAFCFsAAAAA4AKELQAl7uDBg7JYLJo5c6a7S7msmTNnymKx6ODBg+4uxc7UqVNVs2ZNeXt7q3Hjxi59r9WrV8tisejLL7906fvAMadPn1Z4eLhmzZrl7lKczqx/30pav379FB0dbbfMYrFo3LhxLn3f9957TzVq1FBmZqZL3wcoiwhbAJwq75emH374wd2lFGjcuHGyWCy2R2BgoGJjY/Xss88qPT3dKe8xe/ZsvfHGG07Z1sWWLVump556Sq1bt9aMGTP00ksv5RuTF5AceXiic+fO6fXXX1fLli0VEhIif39/XXfddRoyZIh++eUXd5cnSdqwYYPGjRuntLQ0p273zTffVPny5XXffffZLf/uu+/UtWtXVatWTf7+/qpRo4a6d++u2bNnO/X9MzIyNG7cOK1evdqp271STz31lCwWi+699153l+Lx+vXrp6ysLL3//vvuLgUodXzcXQCAsicqKkpnz56Vr6+v22p49913FRQUpNOnT2vZsmV68cUXtXLlSq1fv/6qg8js2bO1Y8cODRs2zDnF/tfKlSvl5eWlf/3rX7JarQWOuf766/Xvf//bbtmYMWMUFBSkZ555xqn1lLS//vpLXbp00ZYtW3TrrbeqV69eCgoK0p49e/T5559r+vTpysrKcneZ2rBhg8aPH69+/fopNDTUKdvMzs7Wm2++qeHDh8vb29u2fO7cubr33nvVuHFjPf7446pQoYJSUlK0du1affDBB+rVq5dT3l+6ELbGjx8vSWrfvr3TtnslDMPQZ599pujoaC1YsECnTp1S+fLl3VqTq5w9e1Y+Pq79dc3f3199+/bVa6+9pscee8xj/2cMYEaELQAlzmKxyN/f36013H333apUqZIk6dFHH1WPHj00b948bdy4UXFxcW6t7XKOHTumgICAywYtSapSpYruv/9+u2WTJ09WpUqV8i33NP369dOPP/6oL7/8Uj169LBbN3HiRI8Pk4VZuHCh/vzzT91zzz12y8eNG6fY2Fht3Lgx3/fi2LFjTnnv3NxcU4TYi61evVq///67Vq5cqc6dO2vevHnq27evu8tyiZL6WXnPPfdoypQpWrVqlW6++eYSeU+gLOA0QgAlrqBrtvr166egoCAdPnxYd9xxh4KCglS5cmU9+eSTysnJsXt9bm6u3njjDdWrV0/+/v6qUqWKHnnkEZ04ceKKa8r75SIlJaXQcf/85z9Vr149+fn5KTIyUoMHD7Y7Xax9+/ZatGiRfv31V9vpepdeg3Gp8+fPa+LEiapVq5b8/PwUHR2tp59+2u76CYvFohkzZujMmTO27V7NNW8HDhzQP/7xD4WFhSkwMFCtWrXSokWLinxdZmambr31VoWEhGjDhg2SHO9HdHS0br31Vn333Xdq0aKF/P39VbNmTX3yySdFvu+mTZu0aNEi9e/fP1/QkiQ/Pz+98sordstWrlypNm3aqFy5cgoNDdXtt9+u3bt3240p6BoZ6X+nm17MYrFoyJAhmj9/vurXry8/Pz/Vq1dPS5YssXvdyJEjJUkxMTG2XuVdi5SYmKibbrpJoaGhCgoKUp06dfT0008Xuf/z589XdHS0atWqZbd8//79at68eYEBPDw83O75mTNn9MQTT6h69ery8/NTnTp19Morr8gwjAL3c9asWbbv+nvvvafKlStLksaPH2/br4uvJfr555919913KywsTP7+/rrhhhv0n//8J19dO3fu1M0336yAgABdc801euGFF5Sbm1vkZ3CxWbNmKTY2Vh06dFDHjh0vex3b22+/rXr16ikwMFAVKlTQDTfckO/0ysOHD6t///6KjIyUn5+fYmJiNGjQILuAmZaWpmHDhtk+u9q1a+vll1+2qzvv59orr7yi6dOn2/4+N2/eXN9//32+2vK+R/7+/qpfv76+/vrrAvfh0s8577u5b98+29HTkJAQPfjgg8rIyLB77dmzZzV06FBVqlRJ5cuX12233abDhw8XeB1Ys2bNFBYWpm+++abAOgBcGY5sATCNnJwcde7cWS1bttQrr7yi5cuX69VXX1WtWrU0aNAg27hHHnlEM2fO1IMPPqihQ4cqJSVF77zzjn788UetX7/+ik5P3L9/vySpYsWKlx0zbtw4jR8/Xh07dtSgQYO0Z88evfvuu/r+++9t7/vMM8/o5MmT+v333/X6669LkoKCggp974cfflgff/yx7r77bj3xxBPatGmTJk2apN27d9t+Afv3v/+t6dOna/Pmzfrwww8lSTfeeGOx91OSjh49qhtvvFEZGRkaOnSoKlasqI8//li33XabvvzyS915550Fvu7s2bO6/fbb9cMPP2j58uVq3ry5pOL1Y9++fbr77rvVv39/9e3bVx999JH69eunZs2aqV69epetOe+X9gceeMChfVy+fLm6du2qmjVraty4cTp79qzefvtttW7dWlu3bi0yAF/Od999p3nz5un//u//VL58eb311lvq0aOHDh06pIoVK+quu+7SL7/8os8++0yvv/667ehp5cqVtXPnTt16661q2LChJkyYID8/P+3bt0/r168v8n03bNigpk2b5lseFRWlFStW6Pfff9c111xz2dcbhqHbbrtNq1atUv/+/dW4cWMtXbpUI0eO1OHDh23f1TwrV67UF198oSFDhqhSpUpq1KiR3n33XQ0aNEh33nmn7rrrLklSw4YNJV0IUK1bt1a1atU0evRolStXTl988YXuuOMOffXVV7bvVGpqqjp06KDz58/bxk2fPl0BAQGONUAXAv9XX32lJ554QpLUs2dPPfjgg0pNTVVERIRt3AcffKChQ4fq7rvv1uOPP65z585p+/bt2rRpk+30yj/++EMtWrRQWlqaBg4cqLp16+rw4cP68ssvlZGRIavVqoyMDLVr106HDx/WI488oho1amjDhg0aM2aMjhw5ku/6zNmzZ+vUqVN65JFHZLFYNGXKFN111106cOCA7e/CsmXL1KNHD8XGxmrSpEn6+++/9eCDDxbaw0vdc889iomJ0aRJk7R161Z9+OGHCg8P18svv2wb069fP33xxRd64IEH1KpVK61Zs0YJCQmX3WbTpk0d+j4CKAYDAJxoxowZhiTj+++/v+yYlJQUQ5IxY8YM27K+ffsakowJEybYjW3SpInRrFkz2/N169YZkoxZs2bZjVuyZEmByy/1/PPPG5KMPXv2GH/++aeRkpJivP/++4afn59RpUoV48yZM3b7kZKSYhiGYRw7dsywWq1GfHy8kZOTY9veO++8Y0gyPvroI9uyhIQEIyoqqtA68iQnJxuSjIcffthu+ZNPPmlIMlauXGlb1rdvX6NcuXIObfdi9erVM9q1a2d7PmzYMEOSsW7dOtuyU6dOGTExMUZ0dLRt/1atWmVIMubOnWucOnXKaNeunVGpUiXjxx9/tL2uOP2IiooyJBlr1661LTt27Jjh5+dnPPHEE4Xuw5133mlIMk6cOOHQPjdu3NgIDw83/v77b9uybdu2GV5eXkafPn1sy/r27Vtgr/K+JxeTZFitVmPfvn1225RkvP3227ZlU6dOtfvu5Hn99dcNScaff/7p0D7kyc7ONiwWS4Gf0b/+9S9bXR06dDCee+45Y926dXbfUcMwjPnz5xuSjBdeeMFu+d13321YLBa7fZJkeHl5GTt37rQb++effxqSjOeffz5fHbfccovRoEED49y5c7Zlubm5xo033mhce+21tmV5371NmzbZlh07dswICQkp8DMryJdffmlIMvbu3WsYhmGkp6cb/v7+xuuvv2437vbbbzfq1atX6Lb69OljeHl5FfjzKjc31zAMw5g4caJRrlw545dffrFbP3r0aMPb29s4dOiQYRj/+7lWsWJF4/jx47Zx33zzjSHJWLBggW1Z48aNjapVqxppaWm2ZcuWLTMk5fs+XvqZ5303H3roIbtxd955p1GxYkXb8y1bthiSjGHDhtmN69ev32X7OHDgQCMgICDfcgBXjtMIAZjKo48+ave8TZs2OnDggO353LlzFRISok6dOumvv/6yPZo1a6agoCCtWrXKofepU6eOKleurJiYGD3yyCOqXbu2Fi1apMDAwALHL1++XFlZWRo2bJi8vP73o3PAgAEKDg526BS8gixevFiSNGLECLvlef/X/kq3W9R7tmjRQjfddJNtWVBQkAYOHKiDBw9q165dduNPnjyp+Ph4/fzzz1q9erXdlPPF7UdsbKzatGlje165cmXVqVPHrscFyZsp0pFJEI4cOaLk5GT169dPYWFhtuUNGzZUp06dbJ/5lejYsaPdqXwNGzZUcHBwkfVLsk2W8c033xTrtLnjx4/LMAxVqFAh37qHHnpIS5YsUfv27fXdd99p4sSJatOmja699lrbaZ7ShZ57e3tr6NChdq9/4oknZBiGvv32W7vl7dq1U2xsrMP1rVy5Uvfcc49OnTpl+w78/fff6ty5s/bu3avDhw/b6mjVqpVatGhhe33lypXVu3dvhz+PWbNm6YYbblDt2rUlXfhOJCQk5DuVMDQ0VL///nuBp/BJF05/nT9/vrp3764bbrgh3/q800jnzp2rNm3aqEKFCnbf8Y4dOyonJ0dr1661e929995r16u873vedyTv+9m3b1+FhITYxnXq1Mnhz1wq+Gfl33//bfu7knd66//93//ZjXvssccuu80KFSro7Nmz+U5HBHDlOI0QgGn4+/vbrgvJU6FCBbtrf/bu3auTJ0/mux4lj6OTAnz11VcKDg6Wr6+vrrnmmnzXwlzq119/lXQhpF3MarWqZs2atvXF9euvv8rLy8v2i2OeiIgIhYaGXvF2i3rPli1b5lt+/fXX29bXr1/ftnzYsGE6d+6cfvzxx3yn+hW3HzVq1Mg35tIeFyQ4OFiSdOrUqSJn+Ltcr6QL+7h06VKdOXNG5cqVK3Q7BbnS+qULv4R/+OGHevjhhzV69Gjdcsstuuuuu3T33XfbBfjLMS65tipP586d1blzZ2VkZGjLli2aM2eO3nvvPd166636+eefFR4erl9//VWRkZH5wurFPb9YTExMkfXk2bdvnwzD0HPPPafnnnuuwDHHjh1TtWrVLvvdK6hXBUlLS9PixYs1ZMgQ7du3z7a8devW+uqrr/TLL7/ouuuukySNGjVKy5cvV4sWLVS7dm3Fx8erV69eat26tSTpzz//VHp6ut13vSB79+7V9u3b8/1sunjfLnbpdyQveOV9R/I+62uvvTbfturUqaOtW7cWWo8j7xMcHGz72XJpLy/9WXOxvO8YsxECzkPYAmAaF09pfTm5ubmF3tj1cr8QXapt27a262nMwMy/3Nx+++36/PPPNXnyZH3yySd2waC4/bhcjy8XJPLUrVtXkvTTTz/ZHRm7Wpf73C+dlCXPldYvSQEBAVq7dq1WrVqlRYsWacmSJZozZ45uvvlmLVu27LLbDgsLk8ViKTLQBQYGqk2bNmrTpo0qVaqk8ePH69tvv72iWfqKcw1V3lG6J598Up07dy5wTGG/4BfH3LlzlZmZqVdffVWvvvpqvvWzZs2yTU9//fXXa8+ePVq4cKGWLFmir776Sv/85z81duxY2xhH5ObmqlOnTnrqqacKXJ8X7vJczXekOFzxPidOnFBgYGCx+g+gcIQtAB6lVq1aWr58uVq3bl2ivxBERUVJkvbs2aOaNWvalmdlZSklJUUdO3a0LStOcIqKilJubq727t1rO8ogXZjEIi0tzfa+zhQVFaU9e/bkW/7zzz/b1l/sjjvuUHx8vPr166fy5cvr3Xffta0rqX50795dkyZN0qefflpk2Lq4V5f6+eefValSJdtRrQoVKhR48+GrOaJYWP+9vLx0yy236JZbbtFrr72ml156Sc8884xWrVpl9x26mI+Pj2rVqlXkTJkXyzst7siRI5IufCbLly/Pdz+qy/W8OPuV9/fB19f3svuQJyoqSnv37s23vKBeFWTWrFmqX7++nn/++Xzr3n//fc2ePdsuSJUrV0733nuv7r33XmVlZemuu+7Siy++qDFjxqhy5coKDg7Wjh07Cn3PWrVq6fTp00Xum6PyPuur+RwcfZ/c3FylpKTYHUW7+IjgpVJSUux+DgG4elyzBcCj3HPPPcrJydHEiRPzrTt//nyBvzg7Q8eOHWW1WvXWW2/Z/Z/jf/3rXzp58qTdDF/lypXTyZMnHdput27dJCnfjGavvfaaJBU6c9iV6tatmzZv3qykpCTbsjNnzmj69OmKjo4u8LqRPn366K233tJ7772nUaNG2ZaXVD/i4uLUpUsXffjhh5o/f36+9VlZWXryySclSVWrVlXjxo318ccf273/jh07tGzZMttnLl34RfrkyZPavn27bdmRI0cuOw23I/KC3KX7fvz48Xxj865/u3ia/4LExcXphx9+yLd8xYoVBY7Puy4t7/S8bt26KScnR++8847duNdff10Wi0Vdu3Yt9P0l2a5nvHS/wsPD1b59e73//vu2cHexP//80/bnbt26aePGjdq8ebPd+ssdGb3Yb7/9prVr1+qee+7R3Xffne/x4IMPat++fdq0aZMk6e+//7Z7vdVqVWxsrAzDUHZ2try8vHTHHXdowYIFBX62eX/P77nnHiUlJWnp0qX5xqSlpen8+fNF1n6xi7+fF/+cSExMzHe95NXIO8r4z3/+027522+/fdnXbN269YpnOQVQMI5sAXCJjz76yO7+Q3kef/zxq9puu3bt9Mgjj2jSpElKTk5WfHy8fH19tXfvXs2dO1dvvvmm7r777qt6j4JUrlxZY8aM0fjx49WlSxfddttt2rNnj/75z3+qefPmdjcMbtasmebMmaMRI0aoefPmCgoKUvfu3QvcbqNGjdS3b19Nnz5daWlpateunTZv3qyPP/5Yd9xxhzp06OD0fRk9erQ+++wzde3aVUOHDlVYWJg+/vhjpaSk6Kuvvrrs9UNDhgxRenq6nnnmGYWEhOjpp58u0X588sknio+P11133aXu3bvrlltuUbly5bR37159/vnnOnLkiO1eW1OnTlXXrl0VFxen/v3726Z+DwkJsbu/0H333adRo0bpzjvv1NChQ5WRkaF3331X1113ncPXzlyqWbNmkqRnnnlG9913n3x9fdW9e3dNmDBBa9euVUJCgqKionTs2DH985//1DXXXGM3WUlBbr/9dv373/+2uyYpb3lMTIy6d++uWrVq6cyZM1q+fLkWLFig5s2b27533bt3V4cOHfTMM8/o4MGDatSokZYtW6ZvvvlGw4YNK/KaRenCqYWxsbGaM2eOrrvuOoWFhal+/fqqX7++pk2bpptuukkNGjTQgAEDVLNmTR09elRJSUn6/ffftW3bNknSU089pX//+9/q0qWLHn/8cdvU71FRUXaBtyCzZ8+2TWFfkG7dusnHx0ezZs1Sy5YtFR8fr4iICLVu3VpVqlTR7t279c477yghIcF2dO+ll17SsmXL1K5dOw0cOFDXX3+9jhw5orlz5+q7775TaGioRo4cqf/85z+69dZbbbcpOHPmjH766Sd9+eWXOnjwYLFPSZ40aZISEhJ000036aGHHtLx48dt9wQ7ffp0sbZ1Oc2aNVOPHj30xhtv6O+//7ZN/f7LL79Iyn+kcsuWLTp+/Lhuv/12p7w/gP9yyxyIAEqtvCnTL/f47bffLjv1e0HTmhc0BbdhGMb06dONZs2aGQEBAUb58uWNBg0aGE899ZTxxx9/FFpf3vaKmn770qnf87zzzjtG3bp1DV9fX6NKlSrGoEGD8k1Hfvr0aaNXr15GaGhogVM5Xyo7O9sYP368ERMTY/j6+hrVq1c3xowZYzeNtmE4b+p3wzCM/fv3G3fffbcRGhpq+Pv7Gy1atDAWLlxoN+biqd8v9tRTTxmSjHfeece2zJF+REVFGQkJCfnqa9euXb76LicjI8N45ZVXjObNmxtBQUGG1Wo1rr32WuOxxx6zm77cMAxj+fLlRuvWrY2AgAAjODjY6N69u7Fr165821y2bJlRv359w2q1GnXq1DE+/fTTy079Pnjw4Hyvj4qKMvr27Wu3bOLEiUa1atUMLy8v2/doxYoVxu23325ERkYaVqvViIyMNHr27JlvSvGCZGZmGpUqVTImTpxot/yzzz4z7rvvPqNWrVpGQECA4e/vb8TGxhrPPPOMkZ6ebjf21KlTxvDhw43IyEjD19fXuPbaa42pU6fapjgvaj8NwzA2bNhgNGvWzLBarfmmD9+/f7/Rp08fIyIiwvD19TWqVatm3HrrrcaXX35pt43t27cb7dq1M/z9/Y1q1aoZEydOtE1hX9jU7w0aNDBq1KhR6OfUvn17Izw83MjOzjbef/99o23btkbFihUNPz8/o1atWsbIkSONkydP2r3m119/Nfr06WNUrlzZ8PPzM2rWrGkMHjzYyMzMtPvsxowZY9SuXduwWq1GpUqVjBtvvNF45ZVXjKysLMMw/jf1+9SpU/PVdelnZRiG8dVXXxnXX3+94efnZ8TGxhrz5s0r8FYEl772cj/DCvqZdebMGWPw4MFGWFiYERQUZNxxxx3Gnj17DEnG5MmT7V4/atQoo0aNGvm+DwCujsUwnHzFJgAAcLqJEydqxowZ2rt3r0OTyQAFSU5OVpMmTfTpp5/aptzPzMxUdHS0Ro8efdVnHwCwxzVbAAB4gOHDh+v06dP6/PPP3V0KPMTZs2fzLXvjjTfk5eWltm3b2pbNmDFDvr6++e7dBeDqcWQLAACgFBo/fry2bNmiDh06yMfHR99++62+/fZbDRw4UO+//767ywPKBMIWAABAKZSYmKjx48dr165dOn36tGrUqKEHHnhAzzzzjHx8mCMNKAmELQAAAABwAa7ZAgAAAAAXcGvYmjRpkpo3b67y5csrPDxcd9xxR767p587d06DBw9WxYoVFRQUpB49eujo0aN2Yw4dOqSEhAQFBgYqPDxcI0eOzHeTwdWrV6tp06by8/NT7dq1NXPmTFfvHgAAAIAyzK0n7K5Zs0aDBw9W8+bNdf78eT399NOKj4/Xrl27VK5cOUkXZl9atGiR5s6dq5CQEA0ZMkR33XWX1q9fL0nKyclRQkKCIiIitGHDBh05ckR9+vSRr6+vXnrpJUlSSkqKEhIS9Oijj2rWrFlasWKFHn74YVWtWtV2h/XC5Obm6o8//lD58uXz3QQQAAAAQNlhGIZOnTqlyMhIeXkVcezKbXf4KsCxY8cMScaaNWsMwzCMtLQ0w9fX1+6Gmrt37zYkGUlJSYZhGMbixYsNLy8vIzU11Tbm3XffNYKDg203JHzqqaeMevXq2b3Xvffea3Tu3Nmhun777bdCb9LKgwcPHjx48ODBgwePsvX47bffiswRppqK5uTJk5KksLAwSdKWLVuUnZ2tjh072sbUrVtXNWrUUFJSklq1aqWkpCQ1aNBAVapUsY3p3LmzBg0apJ07d6pJkyZKSkqy20bemGHDhhVYR2ZmpjIzM23Pjf/OIZKSkqLy5cs7ZV+vRnZ2tlatWqUOHTrI19fX3eWUefTDXOiH+dATc6Ef5kI/zIeemIsZ+3Hq1CnFxMQ4lAtME7Zyc3M1bNgwtW7dWvXr15ckpaamymq1KjQ01G5slSpVlJqaahtzcdDKW5+3rrAx6enpOnv2rAICAuzWTZo0SePHj89XY1JSkgIDA698J50oMDBQmzZtcncZ+C/6YS70w3zoibnQD3OhH+ZDT8zFbP3IyMiQJIcuLzJN2Bo8eLB27Nih7777zt2laMyYMRoxYoTteXp6uqpXr674+HgFBwe7sbILsrOzlZiYqE6dOpkm4Zdl9MNc6If50BNzoR/mQj/Mh56Yixn7kZ6e7vBYU4StIUOGaOHChVq7dq2uueYa2/KIiAhlZWUpLS3N7ujW0aNHFRERYRuzefNmu+3lzVZ48ZhLZzA8evSogoOD8x3VkiQ/Pz/5+fnlW+7r62uaJkvmq6esox/mQj/Mh56YC/0wF/phPvTEXMzUj+LU4dap3w3D0JAhQ/T1119r5cqViomJsVvfrFkz+fr6asWKFbZle/bs0aFDhxQXFydJiouL008//aRjx47ZxiQmJio4OFixsbG2MRdvI29M3jYAAAAAwNncemRr8ODBmj17tr755huVL1/edo1VSEiIAgICFBISov79+2vEiBEKCwtTcHCwHnvsMcXFxalVq1aSpPj4eMXGxuqBBx7QlClTlJqaqmeffVaDBw+2HZ169NFH9c477+ipp57SQw89pJUrV+qLL77QokWL3LbvAAAAAEo3tx7Zevfdd3Xy5Em1b99eVatWtT3mzJljG/P666/r1ltvVY8ePdS2bVtFRERo3rx5tvXe3t5auHChvL29FRcXp/vvv199+vTRhAkTbGNiYmK0aNEiJSYmqlGjRnr11Vf14YcfOnSPLQAAAAC4Em49spU3pXph/P39NW3aNE2bNu2yY6KiorR48eJCt9O+fXv9+OOPxa4RAAAAAK6EW49sAQAAAEBpRdgCAAAAABcgbAEAAACACxC2AAAAAMAFCFsAAAAA4AKELQAAAABwAcIWAAAAALgAYcvD5OQa2pRyXFv+smhTynHl5BZ9rzIAAAAAJc+tNzVG8SzZcUTjF+zSkZPnJHnrk70/qGqIv57vHqsu9au6uzwAAAAAF+HIlodYsuOIBn269b9B639ST57ToE+3asmOI26qDAAAAEBBCFseICfX0PgFu1TQCYN5y8Yv2MUphQAAAICJELY8wOaU4/mOaF3MkHTk5DltTjleckUBAAAAKBRhywMcO3X5oHUl4wAAAAC4HmHLA4SX93fqOAAAAACuR9jyAC1iwhRo9S50TKDVWy1iwkqoIgAAAABFIWx5gJxcQ2ezcwodczY7hwkyAAAAABMhbHmAfycdlFFEjjKMC+MAAAAAmANhywP8ejzDqeMAAAAAuB5hywNEhQU6dRwAAAAA1yNseYAH4qLlZSl8jJflwjgAAAAA5kDY8gBWHy8NaBNT6JgBbWJk9aGdAAAAgFn4uLsAOGZMt1hJ0vR1KXaTZXhZLgStvPUAAAAAzIGw5UHGdItV42sqaNDsrSrvY+ixTnXUr3UtjmgBAAAAJkTY8jB+1gvBKtRPevDGaPkStAAAAABT4jd1D+NluTBTBrcvBgAAAMyNsOVhbGGLtAUAAACYGmHLwxi5F1LWiUxpxoaDyjqf6+aKAAAAABSEsOVBJi3epX4ffy9Jysix6KVvf1Hd577VpMW73FwZAAAAgEsxQYaHmLR4l95fm5Jvea4h23KmfwcAAADMgyNbHiDrfK4+WJc/aF3sg3UpnFIIAAAAmAhhywP8O+mgcouYECPXuDAOAAAAgDkQtjzAr8cznDoOAAAAgOsRtjxAVFigU8cBAAAAcD3Clgd4IC5aXpbCx3hZLowDAAAAYA6ELQ9g9fHSgDYxhY4Z0CZGVh/aCQAAAJgFU797iLxp3T9Yl2I3WYaX5ULQYtp3AAAAwFwIWx5kTLdYdbw+Qv94P0l+Xoae6FxH/VrX4ogWAAAAYEKELQ+TF6wCfaQHb4yWL0ELAAAAMCV+U/cwliImygAAAABgDoQtAAAAAHABwpaHMooeAgAAAMCNCFsexiLOIwQAAAA8AWELAAAAAFyAsAUAAAAALkDY8jDMRggAAAB4BreGrbVr16p79+6KjIyUxWLR/Pnz7dZbLJYCH1OnTrWNiY6Ozrd+8uTJdtvZvn272rRpI39/f1WvXl1Tpkwpid0DAAAAUIa5NWydOXNGjRo10rRp0wpcf+TIEbvHRx99JIvFoh49etiNmzBhgt24xx57zLYuPT1d8fHxioqK0pYtWzR16lSNGzdO06dPd+m+uRqzEQIAAADm5uPON+/atau6du162fURERF2z7/55ht16NBBNWvWtFtevnz5fGPzzJo1S1lZWfroo49ktVpVr149JScn67XXXtPAgQOvficAAAAAoABuDVvFcfToUS1atEgff/xxvnWTJ0/WxIkTVaNGDfXq1UvDhw+Xj8+FXUtKSlLbtm1ltVpt4zt37qyXX35ZJ06cUIUKFfJtLzMzU5mZmbbn6enpkqTs7GxlZ2c7e9eK5fz587Y/u7sWXJDXB/phDvTDfOiJudAPc6Ef5kNPzMWM/ShOLR4Ttj7++GOVL19ed911l93yoUOHqmnTpgoLC9OGDRs0ZswYHTlyRK+99pokKTU1VTExMXavqVKlim1dQWFr0qRJGj9+fL7ly5YtU2BgoLN26Yr8fkaSfCRDSkxMdGstsEc/zIV+mA89MRf6YS70w3zoibmYqR8ZGRkOj/WYsPXRRx+pd+/e8vf3t1s+YsQI258bNmwoq9WqRx55RJMmTZKfn98VvdeYMWPstpuenq7q1asrPj5ewcHBV7YDTrLrSLqmbt8oSerUqZN8fX3dWg8u/N+NxMRE+mES9MN86Im50A9zoR/mQ0/MxYz9yDvrzREeEbbWrVunPXv2aM6cOUWObdmypc6fP6+DBw+qTp06ioiI0NGjR+3G5D2/3HVefn5+BQY1X19ftzc57/RIyRz14H/oh7nQD/OhJ+ZCP8yFfpgPPTEXM/WjOHV4xH22/vWvf6lZs2Zq1KhRkWOTk5Pl5eWl8PBwSVJcXJzWrl1rd25lYmKi6tSpU+AphAAAAADgDG4NW6dPn1ZycrKSk5MlSSkpKUpOTtahQ4dsY9LT0zV37lw9/PDD+V6flJSkN954Q9u2bdOBAwc0a9YsDR8+XPfff78tSPXq1UtWq1X9+/fXzp07NWfOHL355pt2pwl6IqZ+BwAAAMzNracR/vDDD+rQoYPteV4A6tu3r2bOnClJ+vzzz2UYhnr27Jnv9X5+fvr88881btw4ZWZmKiYmRsOHD7cLUiEhIVq2bJkGDx6sZs2aqVKlSho7dqzHTvtukcXdJQAAAABwgFvDVvv27WUYhR+jGThw4GWDUdOmTbVx48Yi36dhw4Zat27dFdUIAAAAAFfCI67ZAgAAAABPQ9jyMBbOIgQAAAA8AmELAAAAAFyAsOWhmI0QAAAAMDfClofhNEIAAADAMxC2AAAAAMAFCFseitMIAQAAAHMjbHkYbmoMAAAAeAbCFgAAAAC4AGHLU3EeIQAAAGBqhC0Pw2yEAAAAgGcgbAEAAACACxC2PBRnEQIAAADmRtjyMJxFCAAAAHgGwhYAAAAAuABhCwAAAABcgLDlYZiNEAAAAPAMhC0AAAAAcAHCFgAAAAC4AGHLQzH1OwAAAGBuhC2Pw0VbAAAAgCcgbAEAAACACxC2PBXnEQIAAACmRtjyMEz9DgAAAHgGwhYAAAAAuABhy0NxFiEAAABgboQtD8NZhAAAAIBnIGwBAAAAgAsQtgAAAADABQhbHsbCdIQAAACARyBsAQAAAIALELY8FLMRAgAAAOZG2PIwnEQIAAAAeAbCFgAAAAC4AGHLQ3EaIQAAAGBuhC0Pw2SEAAAAgGcgbAEAAACACxC2PBXnEQIAAACmRtjyMBbmIwQAAAA8AmELAAAAAFyAsAUAAAAALkDY8lBcsgUAAACYG2HLwzD1OwAAAOAZCFsAAAAA4AKELQAAAABwAbeGrbVr16p79+6KjIyUxWLR/Pnz7db369dPFovF7tGlSxe7McePH1fv3r0VHBys0NBQ9e/fX6dPn7Ybs337drVp00b+/v6qXr26pkyZ4updAwAAAFDGuTVsnTlzRo0aNdK0adMuO6ZLly46cuSI7fHZZ5/Zre/du7d27typxMRELVy4UGvXrtXAgQNt69PT0xUfH6+oqCht2bJFU6dO1bhx4zR9+nSX7RcAAAAA+Ljzzbt27aquXbsWOsbPz08REREFrtu9e7eWLFmi77//XjfccIMk6e2331a3bt30yiuvKDIyUrNmzVJWVpY++ugjWa1W1atXT8nJyXrttdfsQtnFMjMzlZmZaXuenp4uScrOzlZ2dvaV7KrTnD9/XtKF2QjdXQsuyOsD/TAH+mE+9MRc6Ie50A/zoSfmYsZ+FKcWt4YtR6xevVrh4eGqUKGCbr75Zr3wwguqWLGiJCkpKUmhoaG2oCVJHTt2lJeXlzZt2qQ777xTSUlJatu2raxWq21M586d9fLLL+vEiROqUKFCvvecNGmSxo8fn2/5smXLFBgY6IK9dNzxTCmvbYmJiW6tBfboh7nQD/OhJ+ZCP8yFfpgPPTEXM/UjIyPD4bGmDltdunTRXXfdpZiYGO3fv19PP/20unbtqqSkJHl7eys1NVXh4eF2r/Hx8VFYWJhSU1MlSampqYqJibEbU6VKFdu6gsLWmDFjNGLECNvz9PR0Va9eXfHx8QoODnb2bhbL4bSzGr91nSSpU6dO8vX1dWs9uPB/NxITE+mHSdAP86En5kI/zIV+mA89MRcz9iPvrDdHmDps3XfffbY/N2jQQA0bNlStWrW0evVq3XLLLS57Xz8/P/n5+eVb7uvr6/Ym+/j897ClYY568D/0w1zoh/nQE3OhH+ZCP8yHnpiLmfpRnDo8aur3mjVrqlKlStq3b58kKSIiQseOHbMbc/78eR0/ftx2nVdERISOHj1qNybv+eWuBTMzC3c1BgAAADyCR4Wt33//XX///beqVq0qSYqLi1NaWpq2bNliG7Ny5Url5uaqZcuWtjFr1661u5AtMTFRderUKfAUQgAAAABwBreGrdOnTys5OVnJycmSpJSUFCUnJ+vQoUM6ffq0Ro4cqY0bN+rgwYNasWKFbr/9dtWuXVudO3eWJF1//fXq0qWLBgwYoM2bN2v9+vUaMmSI7rvvPkVGRkqSevXqJavVqv79+2vnzp2aM2eO3nzzTbtrsjyR4e4CAAAAABTKrWHrhx9+UJMmTdSkSRNJ0ogRI9SkSRONHTtW3t7e2r59u2677TZdd9116t+/v5o1a6Z169bZXU81a9Ys1a1bV7fccou6deumm266ye4eWiEhIVq2bJlSUlLUrFkzPfHEExo7duxlp303O04iBAAAADyDWyfIaN++vQzj8sdoli5dWuQ2wsLCNHv27ELHNGzYUOvWrSt2fQAAAABwpTzqmi38D6cRAgAAAOZG2PIwTEYIAAAAeAbCFgAAAAC4AGELAAAAAFyAsOVhLMxHCAAAAHgEwhYAAAAAuABhCwAAAABcgLDloZj6HQAAADA3wpaHYep3AAAAwDMQtgAAAADABQhbnorzCAEAAABTI2x5GM4iBAAAADwDYQsAAAAAXICw5aE4ixAAAAAwN8KWp+E8QgAAAMAjELYAAAAAwAUIWwAAAADgAoQtD2PhPEIAAADAIxC2AAAAAMAFCFseyuAIFwAAAGBqhC0PYyFjAQAAAB6BsAUAAAAALkDYAgAAAAAXIGx5GM4iBAAAADwDYQsAAAAAXICw5cEMw3B3CQAAAAAug7DlYSxMRwgAAAB4BMIWAAAAALgAYQsAAAAAXICw5WFycv93ndbGA8ftngMAAAAwD8KWB1my44hufWud7XmfmVt008srtWTHETdWBQAAAKAghC0PsWTHEQ36dKuOnsq0W5568pwGfbqVwAUAAACYDGHLA+TkGhq/YJcKOmEwb9n4Bbs4pRAAAAAwEcKWB9icclxHTp677HpD0pGT57Q55XjJFQUAAACgUIQtD3Ds1OWD1pWMAwAAAOB6hC0PEF7e36njAAAAALgeYcsDtIgJU2igb6FjQgN91SImrIQqAgAAAFAUwlYpYXF3AQAAAADsELY8wOaU40rLyC50zImMbCbIAAAAAEyEsOUBmCADAAAA8DyELQ8QFmB16jgAAAAArkfY8gA/Hz3l1HEAAAAAXI+w5QF+O5Hh1HEAAAAAXI+w5QGqVwhw6jgAAAAArufWsLV27Vp1795dkZGRslgsmj9/vm1ddna2Ro0apQYNGqhcuXKKjIxUnz599Mcff9htIzo6WhaLxe4xefJkuzHbt29XmzZt5O/vr+rVq2vKlCklsXtOUzci2KnjAAAAALieW8PWmTNn1KhRI02bNi3fuoyMDG3dulXPPfectm7dqnnz5mnPnj267bbb8o2dMGGCjhw5Yns89thjtnXp6emKj49XVFSUtmzZoqlTp2rcuHGaPn26S/fNmY5nZDl1HAAAAADX83Hnm3ft2lVdu3YtcF1ISIgSExPtlr3zzjtq0aKFDh06pBo1atiWly9fXhEREQVuZ9asWcrKytJHH30kq9WqevXqKTk5Wa+99poGDhzovJ1xofDy/k4dBwAAAMD13Bq2iuvkyZOyWCwKDQ21Wz558mRNnDhRNWrUUK9evTR8+HD5+FzYtaSkJLVt21ZW6/+mRe/cubNefvllnThxQhUqVMj3PpmZmcrMzLQ9T09Pl3Th1Mbs7MJvLuwKTa4pr4hgPx1Nz5RRwHqLpIgQPzW5prxb6ivr8j5zPntzoB/mQ0/MhX6YC/0wH3piLmbsR3Fq8Ziwde7cOY0aNUo9e/ZUcPD/rk0aOnSomjZtqrCwMG3YsEFjxozRkSNH9Nprr0mSUlNTFRMTY7etKlWq2NYVFLYmTZqk8ePH51u+bNkyBQYGOnO3HNYtwqKP0vPO+rRctMaQIalrlQwtXfKtGypDnkuPxMK96If50BNzoR/mQj/Mh56Yi5n6kZHh+AzgHhG2srOzdc8998gwDL377rt260aMGGH7c8OGDWW1WvXII49o0qRJ8vPzu6L3GzNmjN1209PTVb16dcXHx9sFvZLUTVLTnUf1wuKflZr+v6NuVUP89UzXuupcr4pb6sKF72diYqI6deokX19fd5dT5tEP86En5kI/zIV+mA89MRcz9iPvrDdHmD5s5QWtX3/9VStXriwy7LRs2VLnz5/XwYMHVadOHUVEROjo0aN2Y/KeX+46Lz8/vwKDmq+vr1ubfGvja9S1YTU1HLdEZ7Jy9fKd9XR38yh5e1mKfjFczt3fD9ijH+ZDT8yFfpgL/TAfemIuZupHceow9X228oLW3r17tXz5clWsWLHI1yQnJ8vLy0vh4eGSpLi4OK1du9bu3MrExETVqVOnwFMIzc7byyKrj7ckqcE1IQQtAAAAwKTcGrZOnz6t5ORkJScnS5JSUlKUnJysQ4cOKTs7W3fffbd++OEHzZo1Szk5OUpNTVVqaqqysi5McZ6UlKQ33nhD27Zt04EDBzRr1iwNHz5c999/vy1I9erVS1arVf3799fOnTs1Z84cvfnmm3anCXqSnFxD2Tm5kqRtv6UpJ7egKTMAAAAAuJtbTyP84Ycf1KFDB9vzvADUt29fjRs3Tv/5z38kSY0bN7Z73apVq9S+fXv5+fnp888/17hx45SZmamYmBgNHz7cLkiFhIRo2bJlGjx4sJo1a6ZKlSpp7NixHjPt+8WW7Dii8Qt26XRmjiRpzPxdemvVAT3fPVZd6ld1c3UAAAAALubWsNW+fXsZxuWPzBS2TpKaNm2qjRs3Fvk+DRs21Lp164pdn5ks2XFEgz7dmm/q99ST5zTo06169/6mBC4AAADAREx9zRYuyMk1NH7BrgLvsZW3bPyCXZxSCAAAAJgIYcsDbE45riMnz112vSHpyMlz2pxyvOSKAgAAAFAowpYHOHbq8kHrSsYBAAAAcD3ClgcIL+/v1HEAAAAAXI+w5QFaxIQpNLDwm6eFBvqqRUxYCVUEAAAAoCiErVKCWxsDAAAA5kLY8gCbU44rLSO70DEnMrKZIAMAAAAwEcKWB2CCDAAAAMDzELY8ABNkAAAAAJ6HsOUBWsSEqWqI/2Wvy7JIqhrizwQZAAAAgIkQtjyAt5dFz3ePlZR/Ioy85893j5W3F9NkAAAAAGZB2PIQXepX1bv3N1VEiP2pghEh/nr3/qbqUr+qmyoDAAAAUBAfdxcAx3WpX1WdYiPU5uWV+uPkOY3pcp0eblubI1oAAACACXFky8N4e1kU5HchI9eNKE/QAgAAAEyq2GHr5ptvVlpaWr7l6enpuvnmm51RE4pg+W++yjXcWwcAAACAyyt22Fq9erWysrLyLT937pzWrVvnlKJQOMt/05Yh0hYAAABgVg5fs7V9+3bbn3ft2qXU1FTb85ycHC1ZskTVqlVzbnUoUN6JgwZZCwAAADAth8NW48aNZbFYZLFYCjxdMCAgQG+//bZTi0PBvP57PNIgbQEAAACm5XDYSklJkWEYqlmzpjZv3qzKlSvb1lmtVoWHh8vb29slRcKe5b/HtrhmCwAAADAvh8NWVFSUJCk3N9dlxcAxeRMQkrUAAAAA87qi+2zt3btXq1at0rFjx/KFr7FjxzqlMBQiL2xxaAsAAAAwrWKHrQ8++ECDBg1SpUqVFBERYZsZT7owSx5hy/W8bLMRAgAAADCrYoetF154QS+++KJGjRrlinpQhJxcQ6fPnZck/Zx6SvH1DW5sDAAAAJhQse+zdeLECf3jH/9wRS0owpIdR9R68grt+/OMJOnNlfvVevIKLdlxxM2VAQAAALhUscPWP/7xDy1btswVtaAQS3Yc0aOfblVqeqbd8tT0TD366VYCFwAAAGAyxT6NsHbt2nruuee0ceNGNWjQQL6+vnbrhw4d6rTicEFOrqHR834qdMzoeT+pU2wEpxQCAAAAJlHssDV9+nQFBQVpzZo1WrNmjd06i8VC2HKBjfv/VlpGdqFj0jKytXH/32p9baUSqgoAAABAYYodtlJSUlxRBwqRdOAvh8cRtgAAAABzKPY1W3AHR08N5BRCAAAAwCyKfWTroYceKnT9Rx99dMXFoGBxtSrqnVX7HBoHAAAAwByKHbZOnDhh9zw7O1s7duxQWlqabr75ZqcVhv9pHh0mi0UyCrmLscVyYRwAAAAAcyh22Pr666/zLcvNzdWgQYNUq1YtpxQFe1t+PVFo0JIuBLEtv57g6BYAAABgEk65ZsvLy0sjRozQ66+/7ozN4RLHTp1z6jgAAAAArue0CTL279+v8+fPO2tzuEh4eX+njgMAAADgesU+jXDEiBF2zw3D0JEjR7Ro0SL17dvXaYXhf5pFVZBFUmFnElr+Ow4AAACAORQ7bP344492z728vFS5cmW9+uqrRc5UiCvzfcrxQoOWdCGIfZ9ynPtsAQAAACZR7LC1atUqV9SBQnBTYwAAAMDzFDts5fnzzz+1Z88eSVKdOnVUuXJlpxUFe7lFHdb6r73HTru2EAAAAAAOK/YEGWfOnNFDDz2kqlWrqm3btmrbtq0iIyPVv39/ZWRkuKLGMq9CoNWhcat+PqYcR5MZAAAAAJcqdtgaMWKE1qxZowULFigtLU1paWn65ptvtGbNGj3xxBOuqLHMq1Tez6FxWTmGNux17JRDAAAAAK5V7LD11Vdf6V//+pe6du2q4OBgBQcHq1u3bvrggw/05ZdfuqLGMi8i2PEp3b/68XcXVgIAAADAUcUOWxkZGapSpUq+5eHh4ZxG6CItYsJk9XZs7JlM7nUGAAAAmEGxw1ZcXJyef/55nTt3zrbs7NmzGj9+vOLi4pxaHC7w9rKoVc2KDo2tUoyjYAAAAABcp9izEb755pvq3LmzrrnmGjVq1EiStG3bNvn7+2vp0qVOLxAX3Naomtbu/bvIcY2rh7q+GAAAAABFKnbYql+/vvbu3atZs2bp559/liT17NlTvXv3VkBAgNMLxAUnz2Y7dRwAAAAA1yr2aYSSFBgYqAEDBujVV1/Vq6++qocffviKgtbatWvVvXt3RUZGymKxaP78+XbrDcPQ2LFjVbVqVQUEBKhjx47au3ev3Zjjx4+rd+/eCg4OVmhoqPr376/Tp+3vN7V9+3a1adNG/v7+ql69uqZMmVLsWt0tLMixGQkdHQcAAADAtRwOW1u2bFGHDh2Unp6eb93JkyfVoUMHbdu2rVhvfubMGTVq1EjTpk0rcP2UKVP01ltv6b333tOmTZtUrlw5de7c2e56sd69e2vnzp1KTEzUwoULtXbtWg0cONC2Pj09XfHx8YqKitKWLVs0depUjRs3TtOnTy9Wre7m6IyExZm5EAAAAIDrOHwa4auvvqqbb75ZwcHB+daFhISoU6dOmjp1qj799FOH37xr167q2rVrgesMw9Abb7yhZ599Vrfffrsk6ZNPPlGVKlU0f/583Xfffdq9e7eWLFmi77//XjfccIMk6e2331a3bt30yiuvKDIyUrNmzVJWVpY++ugjWa1W1atXT8nJyXrttdfsQpnZtYgJU2igr9IyLn+aYGigr1rEhJVgVQAAAAAux+GwtWnTJo0ePfqy67t3764PP/zQKUVJUkpKilJTU9WxY0fbspCQELVs2VJJSUm67777lJSUpNDQUFvQkqSOHTvKy8tLmzZt0p133qmkpCS1bdtWVqvVNqZz5856+eWXdeLECVWoUCHfe2dmZiozM9P2PO9oXnZ2trKz3XNNVE6uoYysnELHZGTlKDs7W7lelhKqCpJs3wl3fTdgj36YDz0xF/phLvTDfOiJuZixH8WpxeGwdfjwYZUvX/6y64OCgnTkyBGH37goqampkpTvnl5VqlSxrUtNTVV4eLjdeh8fH4WFhdmNiYmJybeNvHUFha1JkyZp/Pjx+ZYvW7ZMgYGBV7hHV2f3CYuyzhd+s62s87l647Mlur6CUUJV4WKJiYnuLgEXoR/mQ0/MhX6YC/0wH3piLmbqR3HuLexw2KpcubL27NmTL7jk+fnnn1WpUiWH39jMxowZoxEjRtiep6enq3r16oqPjy/wNMqSsGLuNklHixz3hzVST3Rr5PqCYJOdna3ExER16tRJvr6+7i6nzKMf5kNPzIV+mAv9MB96Yi5m7EdBc1hcjsNhq2PHjnrxxRfVpUuXfOsMw9CLL75od8rf1YqIiJAkHT16VFWrVrUtP3r0qBo3bmwbc+zYMbvXnT9/XsePH7e9PiIiQkeP2oeUvOd5Yy7l5+cnP7/8s/r5+vq6rcl/nMxyeJxZvohljTu/H8iPfpgPPTEX+mEu9MN86Im5mKkfxanD4dkIn332Wf30009q2bKlvvjiC23btk3btm3TnDlz1LJlS+3YsUPPPPPMFRVckJiYGEVERGjFihW2Zenp6dq0aZPi4uIkSXFxcUpLS9OWLVtsY1auXKnc3Fy1bNnSNmbt2rV251YmJiaqTp06BZ5CaFZVg61FDyrGOAAAAACu5XDYqlWrlpYvX64zZ87ovvvuU9OmTdW0aVP17NlTGRkZSkxMVO3atYv15qdPn1ZycrKSk5MlXZgUIzk5WYcOHZLFYtGwYcP0wgsv6D//+Y9++ukn9enTR5GRkbrjjjskSddff726dOmiAQMGaPPmzVq/fr2GDBmi++67T5GRkZKkXr16yWq1qn///tq5c6fmzJmjN9980+40QU9w+pxjF+I5Og4AAACAazl8GqEk3XDDDdqxY4eSk5O1d+9eGYah6667znZaX3H98MMP6tChg+15XgDq27evZs6cqaeeekpnzpzRwIEDlZaWpptuuklLliyRv///7iU1a9YsDRkyRLfccou8vLzUo0cPvfXWW7b1ISEhWrZsmQYPHqxmzZqpUqVKGjt2rEdN+y5JO1NPFz2oGOMAAAAAuFaxwlaexo0bX3HAulj79u1lGJefOc9isWjChAmaMGHCZceEhYVp9uzZhb5Pw4YNtW7duiuu0wyyz+c6dRwAAAAA13L4NEK4V0SIf9GDJAX6XVF+BgAAAOBkhC0PUbNyOYfG/X0mSzm53GcLAAAAcDfClodIyzjv0Lhz2bnanHLcxdUAAAAAKAphy0ME+nk7PPbQ30ySAQAAALjbFV3gk5aWps2bN+vYsWPKzbWfkKFPnz5OKQz2WkRXVOKuY0UPlPT55kO6t0WUiysCAAAAUJhih60FCxaod+/eOn36tIKDg2WxWGzrLBYLYctF+t4YrRcX73Zo7O9p51xcDQAAAICiFPs0wieeeEIPPfSQTp8+rbS0NJ04ccL2OH6ca4VcxerjpSCrY+2yenN2KAAAAOBuxf6t/PDhwxo6dKgCAwNdUQ8KcWujag6Na3NdZRdXAgAAAKAoxQ5bnTt31g8//OCKWlCEFtFhTh0HAAAAwHWKfc1WQkKCRo4cqV27dqlBgwby9fW1W3/bbbc5rTjYqxzk59RxAAAAAFyn2GFrwIABkqQJEybkW2exWJSTk3P1VaFAu1LTHR7Xpg6nEgIAAADuVOzTCHNzcy/7IGi51g8HHZuAxNFxAAAAAFznqqatO3eOKcZL0snTZx0at37fXy6uBAAAAEBRih22cnJyNHHiRFWrVk1BQUE6cOCAJOm5557Tv/71L6cXiP/Z82eGQ+MysnN1+tx5F1cDAAAAoDDFDlsvvviiZs6cqSlTpshqtdqW169fXx9++KFTi4O9jOxch8c+/tkWF1YCAAAAoCjFDluffPKJpk+frt69e8vb29u2vFGjRvr555+dWhzsBVq9ix70X9sPOzaZBgAAAADXuKKbGteuXTvf8tzcXGVnZzulKBTs4dY1HR+cw2mEAAAAgDsVO2zFxsZq3bp1+ZZ/+eWXatKkiVOKQsEeaV/L4bFncwwXVgIAAACgKMW+z9bYsWPVt29fHT58WLm5uZo3b5727NmjTz75RAsXLnRFjfgvq4/j2fhsNmELAAAAcKdiH9m6/fbbtWDBAi1fvlzlypXT2LFjtXv3bi1YsECdOnVyRY24iK/FwYFkLQAAAMCtin1k6/fff1ebNm2UmJiYb93GjRvVqlUrpxSGglUJ8dPvaZlFjsuVlJNryNvL0XQGAAAAwJmKfWQrPj5ex48fz7d8/fr16tKli1OKwuX5eDs2I6EhacNebm4MAAAAuEuxw1arVq0UHx+vU6dO2ZatXbtW3bp10/PPP+/U4pBftdAAh8fO3XLIhZUAAAAAKEyxw9aHH36oGjVqqHv37srMzNSqVauUkJCgCRMmaPjw4a6oERcZ2Nbx6d9/+v2ECysBAAAAUJhihy0vLy99/vnn8vX11c0336zbbrtNkyZN0uOPP+6K+nCJm66t7PDYwyeKvrYLAAAAgGs4NEHG9u3b8y0bN26cevbsqfvvv19t27a1jWnYsKFzK4Sd4kx4kcWMhAAAAIDbOBS2GjduLIvFIsP432/vec/ff/99TZ8+XYZhyGKxKCcnx2XFAgAAAICncChspaSkuLoOFEOAj0Vnzzt22Orw8bOqFub4pBoAAAAAnMOhsBUVFeXqOlAMwQG+Onsqy6GxnV5bqV0vJLi4IgAAAACXKvZNjSVp//79euONN7R7925JUmxsrB5//HHVqlXLqcWhYDUrl9NRB8NWxnkXFwMAAACgQMWejXDp0qWKjY3V5s2b1bBhQzVs2FCbNm1SvXr1lJiY6IoacYmHW8cUa3xOLjNlAAAAACWt2Ee2Ro8ereHDh2vy5Mn5lo8aNUqdOnVyWnEoWOvaFSUZkhybmXDt7mPqUK+KS2sCAAAAYK/YR7Z2796t/v3751v+0EMPadeuXU4pCoXz9rKonLfjR6umJv7swmoAAAAAFKTYYaty5cpKTk7Otzw5OVnh4eHOqAkO6BDpeNg69PcZF1YCAAAAoCAOn0Y4YcIEPfnkkxowYIAGDhyoAwcO6MYbb5QkrV+/Xi+//LJGjBjhskJhr0OkoYW/OTY2O4drtgAAAICS5nDYGj9+vB599FE999xzKl++vF599VWNGTNGkhQZGalx48Zp6NChLisU9nyKcUzSsSu7AAAAADiTw2HLMC4cHbFYLBo+fLiGDx+uU6dOSZLKly/vmupQKB8v6Xxu0ePOOTAGAAAAgHMVazZCi8X+GAkhy718LJKjt9E6fe68gvyv6LZqAAAAAK5AsX77vu666/IFrksdP378qgqC40ICrDp32rGbGw/5dLNmPnyjiysCAAAAkKdYYWv8+PEKCQlxVS0optjIYB395S+Hxq7ed8LF1QAAAAC4WLHC1n333cf07ibSMiZMqxwMW5KUk2vI24vpMgAAAICS4PCcdkWdPoiS90CrGsUav3zbERdVAgAAAOBSDoetvNkIYR5WH69iTes+cv42l9UCAAAAwJ7DYSs3N5dTCE3oribVHB6bnskc8AAAAEBJKcatcd0jOjpaFosl32Pw4MGSpPbt2+db9+ijj9pt49ChQ0pISFBgYKDCw8M1cuRInT/v6KTp5vbCnQ3cXQIAAACAApj+xkvff/+9cnJybM937NihTp066R//+Idt2YABAzRhwgTb88DAQNufc3JylJCQoIiICG3YsEFHjhxRnz595Ovrq5deeqlkdsKFAqzexRp/MiNbIYG+LqoGAAAAQB7TH9mqXLmyIiIibI+FCxeqVq1aateunW1MYGCg3Zjg4GDbumXLlmnXrl369NNP1bhxY3Xt2lUTJ07UtGnTlJXl2D2qzM7P2/Ert+6f/p0LKwEAAACQx/RHti6WlZWlTz/9VCNGjLCbHXHWrFn69NNPFRERoe7du+u5556zHd1KSkpSgwYNVKVKFdv4zp07a9CgQdq5c6eaNGmS730yMzOVmZlpe56eni5Jys7OVnZ2tqt2z2F5NeT9t25EeW07nO7Qa39KzTDFPpQml/YD7kU/zIeemAv9MBf6YT70xFzM2I/i1GIxPGiawS+++EK9evXSoUOHFBkZKUmaPn26oqKiFBkZqe3bt2vUqFFq0aKF5s2bJ0kaOHCgfv31Vy1dutS2nYyMDJUrV06LFy9W165d873PuHHjNH78+HzLZ8+ebXeKollkZEtjfvCWHJqb0NCkG3LEmYQAAABA8WVkZKhXr146efKk3Rl1BfGosNW5c2dZrVYtWLDgsmNWrlypW265Rfv27VOtWrWuKGwVdGSrevXq+uuvv4r8QEtCdna2EhMT1alTJ/n6XkhN1z63zOHXVw/20cqRN7uqvDKnoH7AfeiH+dATc6Ef5kI/zIeemIsZ+5Genq5KlSo5FLY85jTCX3/9VcuXL7cdsbqcli1bSpItbEVERGjz5s12Y44ePSpJioiIKHAbfn5+8vPzy7fc19fXNE2W7OsJ9PVSRrZjU7v/ln5eXt4+8vbiRtXOZLbvR1lHP8yHnpgL/TAX+mE+9MRczNSP4tRh+gky8syYMUPh4eFKSEgodFxycrIkqWrVqpKkuLg4/fTTTzp27JhtTGJiooKDgxUbG+uyekvaI+1qFWv88m1HXFQJAAAAAMlDwlZubq5mzJihvn37ysfnfwfj9u/fr4kTJ2rLli06ePCg/vOf/6hPnz5q27atGjZsKEmKj49XbGysHnjgAW3btk1Lly7Vs88+q8GDBxd49MpTDWpfu3jj5/zookoAAAAASB4StpYvX65Dhw7poYcesltutVq1fPlyxcfHq27dunriiSfUo0cPu2u6vL29tXDhQnl7eysuLk7333+/+vTpY3dfrtLA6uNVrCngcyWdzcopchwAAACAK+MR12zFx8eroHk8qlevrjVr1hT5+qioKC1evNgVpZnKmpE3q9XkFQ6Pf2HRTr14Z0MXVgQAAACUXR5xZAuOiQj1L9b4zfuOuqgSAAAAAIStUqZWJcfvA7b37ywXVgIAAACUbYStUmbuo62LNf7P9MyiBwEAAAAoNsJWKRMWZC3W+G6vO36NFwAAAADHEbZKobBAx2+09ufZ/BOPAAAAALh6hK1SaPHQtsUafzIj20WVAAAAAGUXYasUKu6shL3fX+eiSgAAAICyi7BVStUsxqyEO46edWElAAAAQNlE2CqlvizmrIRns3JcVAkAAABQNhG2SqnizkrY96MNLqoEAAAAKJsIW6VYOT9vh8duPpiunFxmJgQAAACchbBViiU0qFqs8St/SnVRJQAAAEDZQ9gqxcbfVr9Y4x/5bKuLKgEAAADKHsJWKRZg9S5Wg3PFRBkAAACAsxC2Srl1T91crPFPz+PoFgAAAOAMhK1SrlpYQLHGf518zEWVAAAAAGULYasMeKVHo2KNP/RXhosqAQAAAMoOwlYZcGezasUa3/aVVS6qBAAAACg7CFtlgLeXRf7elmK95rpnFruoGgAAAKBsIGyVESue6FCs8Vk5hsZ+s8NF1QAAAAClH2GrjCjuRBmS9EnSr8o6n+uCagAAAIDSj7BVhrx9X5Niv+a6Z791QSUAAABA6UfYKkO6Nax6Ra9LOXbGyZUAAAAApR9hqwzx9rLovfubFvt1HV5b7fxiAAAAgFKOsFXGdKlfVc92q13s10WPXuSCagAAAIDSi7BVBj3cts4VvY7ABQAAADiOsFVG7X+p2xW9bsZ3KU6uBAAAACidCFtllLeXRXExFYr9uvELdykn13BBRQAAAEDpQtgqwz56sOUVva7W04udXAkAAABQ+hC2yrAAq7faX1v8o1sS128BAAAARSFslXEz+994xa8lcAEAAACXR9iCDk5OuOLXErgAAACAghG2IInABQAAADgbYQs2BC4AAADAeQhbsLN7Qpcrfm306EXKOp/rxGoAAAAAz0XYgp0Aq7fa1Q694tdf9+y3emZ+stPqAQAAADwVYQv5fPxw66t6/ayNhzmtEAAAAGUeYQsFuprrt/IQuAAAAFCWEbZwWQQuAAAA4MoRtlAoZwUuJs4AAABAWUPYQpEOTk5QsNVyVdu47tlvNWnxLidVBAAAAJgfYQsO2T6hmyqWs17VNt5fm0LgAgAAQJlB2ILDtjzXSQ2qBV/VNt5fm8IphQAAACgTCFsolgWPtdGOcZ2vahvXPfutUtPOOakiAAAAwJwIWyi2IH+fq544o9XkFcxUCAAAgFLN1GFr3Lhxslgsdo+6deva1p87d06DBw9WxYoVFRQUpB49eujo0aN22zh06JASEhIUGBio8PBwjRw5UufPny/pXSmVmBoeAAAAuDxThy1Jqlevno4cOWJ7fPfdd7Z1w4cP14IFCzR37lytWbNGf/zxh+666y7b+pycHCUkJCgrK0sbNmzQxx9/rJkzZ2rs2LHu2JVS6eDkBF3dPIUELgAAAJROpg9bPj4+ioiIsD0qVaokSTp58qT+9a9/6bXXXtPNN9+sZs2aacaMGdqwYYM2btwoSVq2bJl27dqlTz/9VI0bN1bXrl01ceJETZs2TVlZWe7crVIlZXKCerWoflXbiB69SDm5hpMqAgAAANzPx90FFGXv3r2KjIyUv7+/4uLiNGnSJNWoUUNbtmxRdna2OnbsaBtbt25d1ahRQ0lJSWrVqpWSkpLUoEEDValSxTamc+fOGjRokHbu3KkmTZoU+J6ZmZnKzMy0PU9PT5ckZWdnKzs720V76ri8GsxQS57x3a/XiFtq64ZJq654G7WeXqw37m6ghEZVnViZ65mxH2UZ/TAfemIu9MNc6If50BNzMWM/ilOLqcNWy5YtNXPmTNWpU0dHjhzR+PHj1aZNG+3YsUOpqamyWq0KDQ21e02VKlWUmpoqSUpNTbULWnnr89ZdzqRJkzR+/Ph8y5ctW6bAwMCr3CvnSUxMdHcJ+VwbYNHes17SFZ5cOOzL7fpwebIGXO95R7nM2I+yjH6YDz0xF/phLvTDfOiJuZipHxkZGQ6PNXXY6tq1q+3PDRs2VMuWLRUVFaUvvvhCAQEBLnvfMWPGaMSIEbbn6enpql69uuLj4xUcfHX3mXKG7OxsJSYmqlOnTvL19XV3OXa6dZOufW7ZVWzBoh1p3tqcW03jbq3ntLpcycz9KIvoh/nQE3OhH+ZCP8yHnpiLGfuRd9abI0wdti4VGhqq6667Tvv27VOnTp2UlZWltLQ0u6NbR48eVUREhCQpIiJCmzdvtttG3myFeWMK4ufnJz8/v3zLfX19TdNkyXz15Dk4OUE1xyzS1VyCNWvTYX2//y8te7Jj0YNNwqz9KKvoh/nQE3OhH+ZCP8yHnpiLmfpRnDpMP0HGxU6fPq39+/eratWqatasmXx9fbVixQrb+j179ujQoUOKi4uTJMXFxemnn37SsWPHbGMSExMVHBys2NjYEq+/LDkwKUGxVa/uKOAvf2UycQYAAAA8lqnD1pNPPqk1a9bo4MGD2rBhg+688055e3urZ8+eCgkJUf/+/TVixAitWrVKW7Zs0YMPPqi4uDi1atVKkhQfH6/Y2Fg98MAD2rZtm5YuXapnn31WgwcPLvDIFZxr8eNttGNc56veTq2nF+vzjSlOqAgAAAAoOaYOW7///rt69uypOnXq6J577lHFihW1ceNGVa5cWZL0+uuv69Zbb1WPHj3Utm1bRUREaN68ebbXe3t7a+HChfL29lZcXJzuv/9+9enTRxMmTHDXLpU5Qf4+Trn58ej5u7gfFwAAADyKqa/Z+vzzzwtd7+/vr2nTpmnatGmXHRMVFaXFixc7uzQU08HJCar77Lc6dz73qrYTPXqRU8IbAAAA4GqmPrKF0uXnF7qqT1zUVW8nevQiHT5+1gkVAQAAAK5D2EKJmnB7ff3yQteiBxah9ZSVnFYIAAAAUyNsocRZfbycdioggQsAAABmRdiC2zgzcGVd5bVgAAAAgLMRtuBWzgpc1z37rYZ8+p1TtgUAAAA4A2ELbueswLVwx0lFj16k46eznLI9AAAA4GoQtmAKBycnaOnQtk7ZVtMXEhXDtVwAAABwM8IWTKNOZHmnHeUyxBTxAAAAcC/CFkzHmTctZop4AAAAuAthC6Z0cHKCfL0sTtte9OhFOpuV47TtAQAAAEUhbMG09r7UTeufutlp27t+7BL9461Ep20PAAAAKAxhC6ZWLSxABycnyN/HOV/V7//IUvToRfozPdMp2wMAAAAuh7AFj/DzC1318l0Nnba95i8t51ouAAAAuBRhCx7j3hbVtf+lbk7dJjMWAgAAwFUIW/Ao3l4WHZycoNsaVHXaNvNmLGQCDQAAADgTYQse6a3eTfXLC12dus3rxy7R/e+vdeo2AQAAUHYRtuCxrD5eOjg5QYuHtHHaNr9LOaXo0YuUmnbOadsEAABA2UTYgseLvSbYqTdClqRWk1coevQinT533qnbBQAAQNlB2EKpcXBygoKs3k7dZv1xS5lEAwAAAFeEsIVSZceELvr+6Y5O327eJBo5uYbTtw0AAIDSibCFUqdysJ8OTk5Q86gKTt92racX66N1vzh9uwAAACh9CFsoteYOulG7J3Rx+nYnLNrLDZEBAABQJMIWSrUAq7cOTk5QXExFp287evQifbjqAKcWAgAAoECELZQJnz3SyiVHuV5Yult1n0/UpqMWp28bAAAAno2whTIj7yjXqhHtnb7t2Qe81OS5ZU7fLgAAADwXYQtlTkx4OR2cnCDnHouy6LTEtVwAAACwIWyhzEpx0VGu6NGLtOXACadvFwAAAJ6FsIUyLe8o1/Jh7Zy63R7TNyh69CLtSz3t1O0CAADAcxC2AEm1I4J0cHKC5j/a2qnb7fjGGkIXAABAGUXYAi7SODpUBycnaO2THZy63bzQdTYrx6nbBQAAgHkRtoAC1KgUqIOTE+Tl5Bndrx+7RM3GEboAAADKAsIWUIgDkxLU7trKTt3m3+cuhK7+Mzc5dbsAAAAwF8IWUISP+7fQ7gld1Kl2oFO3u+Lnv9TwaaaKBwAAKK0IW4ADAqze+uDhDjo4OcGp203PvTBV/OHjZ526XQAAALgfYQsopoOTE3RHo2pO3WbrKSsVPXqRUo6dcep2AQAA4D6ELeAKvNGzsX55oavTt9vhtdXMWggAAFBKELaAK2T18dLByQlqUi1YkuHUbV8/donufWeFU7cJAACAkkXYAq7SF4+20tQWOfJx8jTxm34/p+jRTKABAADgqQhbgBNYvaXdE+L1/dMdnb7t6NGLtPGXv52+XQAAALgWYQtwosrBfjo4OUEbR9/i1O3e99FGRY9epH2pp526XQAAALgOYQtwgYhQfx2cnKCvBt7o1O12fGONokcv0oaf/3LqdgEAAOB8hC3AhZrVrOD0e3NJUq+ZmxQ9epGSD6Y5fdsAAABwDsIWUAIOTk7QDTVCnb7dO95bz+mFAAAAJkXYAkrIl//XWrsndHHJtvNOLzz0V4ZLtg8AAIDiI2wBJSjA6q2DkxO0akR7l2y/7SurFD16kTbvO+6S7QMAAMBxpg5bkyZNUvPmzVW+fHmFh4frjjvu0J49e+zGtG/fXhaLxe7x6KOP2o05dOiQEhISFBgYqPDwcI0cOVLnz58vyV0B7MSEl9PByQla/9TNLtn+PR8mKXr0Im05cMIl2wcAAEDRfNxdQGHWrFmjwYMHq3nz5jp//ryefvppxcfHa9euXSpXrpxt3IABAzRhwgTb88DAQNufc3JylJCQoIiICG3YsEFHjhxRnz595Ovrq5deeqlE9we4VLWwAB2cnKDG45cp7Wy207ffY/oGSZK3RVr6eDvVjghy+nsAAACgYKYOW0uWLLF7PnPmTIWHh2vLli1q27atbXlgYKAiIiIK3MayZcu0a9cuLV++XFWqVFHjxo01ceJEjRo1SuPGjZPVanXpPgCOSH4+XsdPZ6npC4ku2X6OceG6LklaNaK9YsLLFfEKAAAAXC1Th61LnTx5UpIUFhZmt3zWrFn69NNPFRERoe7du+u5556zHd1KSkpSgwYNVKVKFdv4zp07a9CgQdq5c6eaNGmS730yMzOVmZlpe56eni5Jys7OVna2848+FFdeDWaoBc7rR3k/i/ZOjNdby/bq7XUpziitQB1eWy1JurZSoD4b0FIhgb4uey934O+H+dATc6Ef5kI/zIeemIsZ+1GcWiyGYRgurMVpcnNzddtttyktLU3fffedbfn06dMVFRWlyMhIbd++XaNGjVKLFi00b948SdLAgQP166+/aunSpbbXZGRkqFy5clq8eLG6du2a773GjRun8ePH51s+e/Zsu1MUAVfJNaQnNlqUKy9JFhe+kyHJUOtKhu6qZcjH1FdxAgAAuF9GRoZ69eqlkydPKjg4uNCxHhO2Bg0apG+//VbfffedrrnmmsuOW7lypW655Rbt27dPtWrVuqKwVdCRrerVq+uvv/4q8gMtCdnZ2UpMTFSnTp3k61u6jkp4Ilf2IzXtnNq8utap2yyKl6T/DIpTncjyJfq+zsLfD/OhJ+ZCP8yFfpgPPTEXM/YjPT1dlSpVcihsecRphEOGDNHChQu1du3aQoOWJLVs2VKSbGErIiJCmzdvthtz9OhRSbrsdV5+fn7y8/PLt9zX19c0TZbMV09Z54p+VK/sq4OTEyRJN09dpQN/u/4+WrmSbn03yfa8Y51wvd27qQKs3i5/b2fi74f50BNzoR/mQj/Mh56Yi5n6UZw6TB22DMPQY489pq+//lqrV69WTExMka9JTk6WJFWtWlWSFBcXpxdffFHHjh1TeHi4JCkxMVHBwcGKjY11We2As60c2UGnz51Xg3FLVZKHo5fvOabrx9pPVjP/0dZqHB1aglUAAAB4HlOHrcGDB2v27Nn65ptvVL58eaWmpkqSQkJCFBAQoP3792v27Nnq1q2bKlasqO3bt2v48OFq27atGjZsKEmKj49XbGysHnjgAU2ZMkWpqal69tlnNXjw4AKPXgFmFuTvo5TJCUpNO6dWk1e4rY473ltv+zPBCwAAoGCmDlvvvvuupAs3Lr7YjBkz1K9fP1mtVi1fvlxvvPGGzpw5o+rVq6tHjx569tlnbWO9vb21cOFCDRo0SHFxcSpXrpz69u1rd18uwNNEhPrr4OQEHT5+Vq2nrHRrLRcHrzyBvl5KHN5e1cICSr4gAAAAkzB12Cpq7o7q1atrzZo1RW4nKipKixcvdlZZgGnk3RQ563yu2r2yQkfSstxdkiQpIzs3Xwh8tvP1erBdjLy9XDm7IgAAgHmYOmwBcIzVx0tJozvpbFaOhn6yVon7XD+RRnG9sHS3Xli6u8B1BDEAAFAaEbaAUiTA6q0PHu4gSW6/rqs4LhfEZvdrqRvrVnJDRQAAAFePsAWUUma6rutK9Zq5yeGxXzwcpxa1w1xYDQAAQPEQtoBSLu+6Lkn66dBJdf/nd26uyDXu+TDpomdeejxpWYHjrq1cTl8Oaq2QQHPcqwMAAJRehC2gDGlQI8QWvDbvO35JQClNvC67Zu+fZ9RoQsFBrCifP9RKra6reKVFAQCAMoawBZRRLWqH2YLXn+mZavPScp1zc01md99HG4v9Gu5DBgBA2UXYAqDKwX76+b/BS5K2HDihHtM3uLGi0qOg+5A5wtdLWjasvWLCyzm5IgAAUFIIWwDyaVazgu2olySdPndeD0xL1I9/5rqxqrIlO1fq8Npqh8Y2rR6qT/q3VJA/P9IBADAT/mUGUKQgfx99/URXSTL1vbzKqq2/pan+uKUOj/9q4I1qVrOCCysCAAASYQtAMV18L6+LpRw74/CRGLhXUaeIEsYAAHAOwhYAp4gJL2d36qEkbfzl7yuaVALuVZzr9QhmAABcHmELgMu0uq5ivgCWhyBWOjgSzOb2b1EClQAAYD6ELQBucbkgtuHnv9Rr5iY3VARX+ce/NquwG03n4SgZAKC0IWwBMJUb61a67NGwi2Wdz9UzC37Q3E1/FrA2V4Xd2BjuUHQ/HDlK5m2Rlj7eTrUjgpxRFAAALkXYAuCRrD5emnpnC0290355dna2Fi9erG7dusjX11eS9NOhk+r+z+/cUCWcLceQOr6xptiv61gnXG/3bqoAq7cLqgIAoGCELQClXoMaIQ4dLbtUTq6hT9b9ovHf7nNBVShJy/cc0/Vjlzg8fv6jrdU4OtR1BQEAygTCFgBchreXRQ+2q6MH29Vx+DXcALp0uOO99Vf0Oj9vixKHt1eNSoHOLQgA4JEIWwDgRBffALq4Dh8/q9ZTVjq5IpSkzBxDbV9Z5fD4ptVD9Un/lgry559jACiN+OkOACZRLSzA4dMduQ6tdNj6W5rqj1vqlG0xeQgAmA9hCwA8UHGuQzublaOhn6xV4r4MF1cFdyre5CH5p+K3SFo0pI1irwl2em0AUFYRtgCglAuweuuDhztcdj1hrCzKPxW/IanbO+uuesvB/t5aNqy9IkL9r3pbAODpCFsAUMYVFcYuRjBDUdLP5ajV5BVX/HrCGoDShLAFAHCYI8Gs4BkZudE0HHO1Ya0o9zevobG315PVh+8jANcjbAEAnOrSGRkLutG0xFEyuMen3x/Sp98fuurtfPFwnFrUDnNCRQBKM8IWAMAtHD19MTXtnEuPdABX4p4Pk5y4tfwTlkhStRB/LXisjcKCrE58LwAlibAFADC1iFB/h2devNiWAyfUY/oGF1QEOFvBpzQePnlOTV9IdMo7dKwTrrd7N1WA1dsp2wPgGMIWAKBUalazgsMh7fS58+rzz+XaeizHxVUB7rF8zzFdP3ZJib4nAQ8gbAEAoCB/H80b0eWKXnv4+Fm1nrLSyRUBns8VAc9L0rdD26pOZHmnbhdwFcIWAABXoVpYQLFOc0w+mKY73lvvwoqA0itXUue31pbAO/3vOrpAXy8lDm+vamEBJfC+KG0IWwAAlKDG0aFXdA1aQa588hCm4gcK97+/HxnZuS45er3g/25SgxohTt8uzIWwBQCAh7qSyUMunor/VKahji8l6nhu0a8D4Fzd//mdy9+D+8q5H2ELAIAyKizIqq0vXf1Rtn2pp9XxjTVOqAiAMznjvnLzH22txtGhzimoDCJsAQCAq1I7IuiqTo0krAHm5YxrTCuV89W3j7dT5WA/J1TkWQhbAADAra42rBVl877jTr4JMYDi+OtMtpq/tLzIcaXxdgGELQAAUKq1qB121WEu63yunlnwg+Zu+tNJVV2MCUsAqbDbBXjpnb3f6cv/u0khgb4lXtfVIGwBAAAUwerjpal3ttDUO5273YsnLPH19dWu39PV7Z11zn0TwON5ae9fGWo04cJ0/Fuf7aSwIKuba3IMYQsAAMAkYq8JdtoplVsOnFCP6Rucsi3ATJq+kKjKQVZ9/2wnd5dSJMIWAABAKdSsZgWXXgtXEAIeSsqfp7PU/IVE0wcuwhYAAACcwhUB78/0TLV5abnOOXWrRTEkWUr0HVF8f57O0vHTWaY+pZCwBQAAANOqHOynn0vwCF12drZmzl2sF7fza7InuPPtVVozprO7y7gsvkUAAADARcLLSXsnxsvX13kz353MyNbtry7TwTNO2yQk/XryvLtLKBRhCwAAAHCxkEBfrX7O9UfouK+cuRC2AAAAgFLiau8rd/rceT0wLVE//pnrxKpcJzTA3HHG3NUBAAAAKDFB/j76+omuV72dPX+cUue31jqhosJNvquhy9/japSpsDVt2jRNnTpVqampatSokd5++221aNHC3WUBAAAApUqdyPIOHWG7mtsFWCR1qhdxRa8tKWUmbM2ZM0cjRozQe++9p5YtW+qNN95Q586dtWfPHoWHh7u7PAAAAKDMKex2Ae+v2q9JS3/W5abif/f+pvL2MvcU/V7uLqCkvPbaaxowYIAefPBBxcbG6r333lNgYKA++ugjd5cGAAAA4BKPdKild+5rpOBLJoWsEuyn9+5vqi71q7qnsGIoE0e2srKytGXLFo0ZM8a2zMvLSx07dlRSUv7ZWjIzM5WZmWl7np6eLunCfReys7NdX3AR8mowQy2gH2ZDP8yHnpgL/TAX+mE+9MRcbr4uTOOb5ajCdTfo+NkchZf30w1RFeTtZXFbj4rzvhbDMAwX1mIKf/zxh6pVq6YNGzYoLi7Otvypp57SmjVrtGnTJrvx48aN0/jx4/NtZ/bs2QoMDHR5vQAAAADMKSMjQ7169dLJkycVHBxc6NgycWSruMaMGaMRI0bYnqenp6t69eqKj48v8gMtCdnZ2UpMTFSnTp2cerM9XBn6YS70w3zoibnQD3OhH+ZDT8zFjP3IO+vNEWUibFWqVEne3t46evSo3fKjR48qIiL/DCZ+fn7y8/PLt9zX19c0TZbMV09ZRz/MhX6YDz0xF/phLvTDfOiJuZipH8Wpo0xMkGG1WtWsWTOtWLHCtiw3N1crVqywO60QAAAAAJylTBzZkqQRI0aob9++uuGGG9SiRQu98cYbOnPmjB588EF3lwYAAACgFCozYevee+/Vn3/+qbFjxyo1NVWNGzfWkiVLVKVKFXeXBgAAAKAUKjNhS5KGDBmiIUOGuLsMAAAAAGVAmbhmCwAAAABKGmELAAAAAFyAsAUAAAAALkDYAgAAAAAXIGwBAAAAgAsQtgAAAADABQhbAAAAAOACZeo+W1fKMAxJUnp6upsruSA7O1sZGRlKT0+Xr6+vu8sp8+iHudAP86En5kI/zIV+mA89MRcz9iMvE+RlhMIQthxw6tQpSVL16tXdXAkAAAAAMzh16pRCQkIKHWMxHIlkZVxubq7++OMPlS9fXhaLxd3lKD09XdWrV9dvv/2m4OBgd5dT5tEPc6Ef5kNPzIV+mAv9MB96Yi5m7IdhGDp16pQiIyPl5VX4VVkc2XKAl5eXrrnmGneXkU9wcLBpvnSgH2ZDP8yHnpgL/TAX+mE+9MRczNaPoo5o5WGCDAAAAABwAcIWAAAAALgAYcsD+fn56fnnn5efn5+7S4Hoh9nQD/OhJ+ZCP8yFfpgPPTEXT+8HE2QAAAAAgAtwZAsAAAAAXICwBQAAAAAuQNgCAAAAABcgbAEAAACACxC2PMy0adMUHR0tf39/tWzZUps3b3Z3SR5v0qRJat68ucqXL6/w8HDdcccd2rNnj92Yc+fOafDgwapYsaKCgoLUo0cPHT161G7MoUOHlJCQoMDAQIWHh2vkyJE6f/683ZjVq1eradOm8vPzU+3atTVz5kxX716pMHnyZFksFg0bNsy2jJ6UrMOHD+v+++9XxYoVFRAQoAYNGuiHH36wrTcMQ2PHjlXVqlUVEBCgjh07au/evXbbOH78uHr37q3g4GCFhoaqf//+On36tN2Y7du3q02bNvL391f16tU1ZcqUEtk/T5KTk6PnnntOMTExCggIUK1atTRx4kRdPN8V/XCttWvXqnv37oqMjJTFYtH8+fPt1pfk5z937lzVrVtX/v7+atCggRYvXuz0/TW7wvqRnZ2tUaNGqUGDBipXrpwiIyPVp08f/fHHH3bboB/OVdTfkYs9+uijslgseuONN+yWl5qeGPAYn3/+uWG1Wo2PPvrI2LlzpzFgwAAjNDTUOHr0qLtL82idO3c2ZsyYYezYscNITk42unXrZtSoUcM4ffq0bcyjjz5qVK9e3VixYoXxww8/GK1atTJuvPFG2/rz588b9evXNzp27Gj8+OOPxuLFi41KlSoZY8aMsY05cOCAERgYaIwYMcLYtWuX8fbbbxve3t7GkiVLSnR/Pc3mzZuN6Ohoo2HDhsbjjz9uW05PSs7x48eNqKgoo1+/fsamTZuMAwcOGEuXLjX27dtnGzN58mQjJCTEmD9/vrFt2zbjtttuM2JiYoyzZ8/axnTp0sVo1KiRsXHjRmPdunVG7dq1jZ49e9rWnzx50qhSpYrRu3dvY8eOHcZnn31mBAQEGO+//36J7q/Zvfjii0bFihWNhQsXGikpKcbcuXONoKAg480337SNoR+utXjxYuOZZ54x5s2bZ0gyvv76a7v1JfX5r1+/3vD29jamTJli7Nq1y3j22WcNX19f46effnL5Z2AmhfUjLS3N6NixozFnzhzj559/NpKSkowWLVoYzZo1s9sG/XCuov6O5Jk3b57RqFEjIzIy0nj99dft1pWWnhC2PEiLFi2MwYMH257n5OQYkZGRxqRJk9xYVelz7NgxQ5KxZs0awzAu/KD29fU15s6daxuze/duQ5KRlJRkGMaFHypeXl5Gamqqbcy7775rBAcHG5mZmYZhGMZTTz1l1KtXz+697r33XqNz586u3iWPderUKePaa681EhMTjXbt2tnCFj0pWaNGjTJuuummy67Pzc01IiIijKlTp9qWpaWlGX5+fsZnn31mGIZh7Nq1y5BkfP/997Yx3377rWGxWIzDhw8bhmEY//znP40KFSrY+pP33nXq1HH2Lnm0hIQE46GHHrJbdtdddxm9e/c2DIN+lLRLf5Esyc//nnvuMRISEuzqadmypfHII484dR89SWG/2OfZvHmzIcn49ddfDcOgH652uZ78/vvvRrVq1YwdO3YYUVFRdmGrNPWE0wg9RFZWlrZs2aKOHTvalnl5ealjx45KSkpyY2Wlz8mTJyVJYWFhkqQtW7YoOzvb7rOvW7euatSoYfvsk5KS1KBBA1WpUsU2pnPnzkpPT9fOnTttYy7eRt4Y+nd5gwcPVkJCQr7PjZ6UrP/85z+64YYb9I9//EPh4eFq0qSJPvjgA9v6lJQUpaam2n2WISEhatmypV0/QkNDdcMNN9jGdOzYUV5eXtq0aZNtTNu2bWW1Wm1jOnfurD179ujEiROu3k2PceONN2rFihX65ZdfJEnbtm3Td999p65du0qiH+5Wkp8/P8OuzMmTJ2WxWBQaGiqJfrhDbm6uHnjgAY0cOVL16tXLt7409YSw5SH++usv5eTk2P3iKElVqlRRamqqm6oqfXJzczVs2DC1bt1a9evXlySlpqbKarXafijnufizT01NLbA3eesKG5Oenq6zZ8+6Ync82ueff66tW7dq0qRJ+dbRk5J14MABvfvuu7r22mu1dOlSDRo0SEOHDtXHH38s6X+fZ2E/n1JTUxUeHm633sfHR2FhYcXqGaTRo0frvvvuU926deXr66smTZpo2LBh6t27tyT64W4l+flfbgz9ubxz585p1KhR6tmzp4KDgyXRD3d4+eWX5ePjo6FDhxa4vjT1xKfE3gnwAIMHD9aOHTv03XffubuUMu23337T448/rsTERPn7+7u7nDIvNzdXN9xwg1566SVJUpMmTbRjxw6999576tu3r5urK3u++OILzZo1S7Nnz1a9evWUnJysYcOGKTIykn4AhcjOztY999wjwzD07rvvurucMmvLli168803tXXrVlksFneX43Ic2fIQlSpVkre3d77Z1o4ePaqIiAg3VVW6DBkyRAsXLtSqVat0zTXX2JZHREQoKytLaWlpduMv/uwjIiIK7E3eusLGBAcHKyAgwNm749G2bNmiY8eOqWnTpvLx8ZGPj4/WrFmjt956Sz4+PqpSpQo9KUFVq1ZVbGys3bLrr79ehw4dkvS/z7Own08RERE6duyY3frz58/r+PHjxeoZpJEjR9qObjVo0EAPPPCAhg8fbjsKTD/cqyQ//8uNoT/55QWtX3/9VYmJibajWhL9KGnr1q3TsWPHVKNGDdu/8b/++queeOIJRUdHSypdPSFseQir1apmzZppxYoVtmW5ublasWKF4uLi3FiZ5zMMQ0OGDNHXX3+tlStXKiYmxm59s2bN5Ovra/fZ79mzR4cOHbJ99nFxcfrpp5/sfjDk/TDP+yU1Li7Obht5Y+hffrfccot++uknJScn2x433HCDevfubfszPSk5rVu3znc7hF9++UVRUVGSpJiYGEVERNh9lunp6dq0aZNdP9LS0rRlyxbbmJUrVyo3N1ctW7a0jVm7dq2ys7NtYxITE1WnTh1VqFDBZfvnaTIyMuTlZf/Pt7e3t3JzcyXRD3cryc+fn2GOyQtae/fu1fLly1WxYkW79fSjZD3wwAPavn273b/xkZGRGjlypJYuXSqplPWkxKbiwFX7/PPPDT8/P2PmzJnGrl27jIEDBxqhoaF2s62h+AYNGmSEhIQYq1evNo4cOWJ7ZGRk2MY8+uijRo0aNYyVK1caP/zwgxEXF2fExcXZ1udNMx4fH28kJycbS5YsMSpXrlzgNOMjR440du/ebUybNo1pxovh4tkIDYOelKTNmzcbPj4+xosvvmjs3bvXmDVrlhEYGGh8+umntjGTJ082QkNDjW+++cbYvn27cfvttxc41XWTJk2MTZs2Gd99951x7bXX2k3jm5aWZlSpUsV44IEHjB07dhiff/65ERgYyFTjl+jbt69RrVo129Tv8+bNMypVqmQ89dRTtjH0w7VOnTpl/Pjjj8aPP/5oSDJee+0148cff7TNbldSn//69esNHx8f45VXXjF2795tPP/882VyqvHC+pGVlWXcdtttxjXXXGMkJyfb/Tt/8Sx29MO5ivo7cqlLZyM0jNLTE8KWh3n77beNGjVqGFar1WjRooWxceNGd5fk8SQV+JgxY4ZtzNmzZ43/+7//MypUqGAEBgYad955p3HkyBG77Rw8eNDo2rWrERAQYFSqVMl44oknjOzsbLsxq1atMho3bmxYrVajZs2adu+Bwl0atuhJyVqwYIFRv359w8/Pz6hbt64xffp0u/W5ubnGc889Z1SpUsXw8/MzbrnlFmPPnj12Y/7++2+jZ8+eRlBQkBEcHGw8+OCDxqlTp+zGbNu2zbjpppsMPz8/o1q1asbkyZNdvm+eJj093Xj88ceNGjVqGP7+/kbNmjWNZ555xu4XR/rhWqtWrSrw342+ffsahlGyn/8XX3xhXHfddYbVajXq1atnLFq0yGX7bVaF9SMlJeWy/86vWrXKtg364VxF/R25VEFhq7T0xGIYF91yHgAAAADgFFyzBQAAAAAuQNgCAAAAABcgbAEAAACACxC2AAAAAMAFCFsAAAAA4AKELQAAAABwAcIWAAAAALgAYQsAAAAAXICwBQAoldq3b69hw4a5uwxTmDlzpkJDQ91dBgCUOYQtAECx/Pnnnxo0aJBq1KghPz8/RUREqHPnzlq/fv1Vb7skA9LMmTNlsVjyPfz9/Yu1HYvFovnz57umyBLYPgDAdXzcXQAAwLP06NFDWVlZ+vjjj1WzZk0dPXpUK1as0N9//33F28zKypLVanVilY4JDg7Wnj177JZZLBanv4+79g8A4F4c2QIAOCwtLU3r1q3Tyy+/rA4dOigqKkotWrTQmDFjdNttt9nGHTp0SLfffruCgoIUHByse+65R0ePHrWtHzdunBo3bqwPP/xQMTEx8vf3V79+/bRmzRq9+eabtqNMBw8elCTt2LFDXbt2VVBQkKpUqaIHHnhAf/31l217Z86cUZ8+fRQUFKSqVavq1VdfdWh/LBaLIiIi7B5VqlSxrW/fvr2GDh2qp556SmFhYYqIiNC4ceNs66OjoyVJd955pywWi+15Qfv3ySefqGLFisrMzLSr4Y477tADDzzgUL0HDx6UxWLRvHnz1KFDBwUGBqpRo0ZKSkqyGzdz5kzVqFFDgYGBuvPOOwsMwt98842aNm0qf39/1axZU+PHj9f58+clSRMmTFBkZKTd6xISEtShQwfl5uY6VCsAgLAFACiGoKAgBQUFaf78+flCQ57c3FzdfvvtOn78uNasWaPExEQdOHBA9957r924ffv26auvvtK8efOUnJysN998U3FxcRowYICOHDmiI0eOqHr16kpLS9PNN9+sJk2a6IcfftCSJUt09OhR3XPPPbZtjRw5UmvWrNE333yjZcuWafXq1dq6datT9vnjjz9WuXLltGnTJk2ZMkUTJkxQYmKiJOn777+XJM2YMUNHjhyxPS9o//7xj38oJydH//nPf2xjjh07pkWLFumhhx4qVk3PPPOMnnzySSUnJ+u6665Tz549bUFp06ZN6t+/v4YMGaLk5GR16NBBL7zwgt3r161bpz59+ujxxx/Xrl279P7772vmzJl68cUXbduPjo7Www8/LEmaNm2aNmzYoI8//lheXvzqAAAOMwAAKIYvv/zSqFChguHv72/ceOONxpgxY4xt27bZ1i9btszw9vY2Dh06ZFu2c+dOQ5KxefNmwzAM4/nnnzd8fX2NY8eO2W27Xbt2xuOPP263bOLEiUZ8fLzdst9++82QZOzZs8c4deqUYbVajS+++MK2/u+//zYCAgLybetiM2bMMCQZ5cqVs3t06dLFrp6bbrrJ7nXNmzc3Ro0aZXsuyfj666/txlxu/wYNGmR07drV9vzVV181atasaeTm5l62zou3n5KSYkgyPvzwQ9v6vM929+7dhmEYRs+ePY1u3brZbePee+81QkJCbM9vueUW46WXXrIb8+9//9uoWrWq7fn+/fuN8uXLG6NGjTICAgKMWbNmXbZGAEDBuGYLAFAsPXr0UEJCgtatW6eNGzfq22+/1ZQpU/Thhx+qX79+2r17t6pXr67q1avbXhMbG6vQ0FDt3r1bzZs3lyRFRUWpcuXKRb7ftm3btGrVKgUFBeVbt3//fp09e1ZZWVlq2bKlbXlYWJjq1KlT5LbLly+f7whYQECA3fOGDRvaPa9ataqOHTtW5LYL2r8BAwaoefPmOnz4sKpVq6aZM2eqX79+xb5O7OKaqlatKunCUbK6detq9+7duvPOO+3Gx8XFacmSJbbn27Zt0/r1621HsiQpJydH586dU0ZGhgIDA1WzZk298soreuSRR3TvvfeqV69exaoRAMAEGQCAK+Dv769OnTqpU6dOeu655/Twww/r+eefV79+/RzeRrly5Rwad/r0aXXv3l0vv/xyvnVVq1bVvn37HH7PS3l5eal27dqFjvH19bV7brFYHLpuqaD9a9KkiRo1aqRPPvlE8fHx2rlzpxYtWlS8oi+pKS+oFedaqtOnT2v8+PG666678q27eDbGtWvXytvbWwcPHtT58+fl48OvDQBQHJx4DQC4arGxsTpz5owk6frrr9dvv/2m3377zbZ+165dSktLU2xsbKHbsVqtysnJsVvWtGlT7dy5U9HR0apdu7bdo1y5cqpVq5Z8fX21adMm22tOnDihX375xYl7eHm+vr75ai7Mww8/rJkzZ2rGjBnq2LGj3RFAZ7j++uvtPgtJ2rhxo93zpk2bas+ePfk+z9q1a9uuyZozZ47mzZun1atX69ChQ5o4caJT6wSAsoCwBQBw2N9//62bb75Zn376qbZv366UlBTNnTtXU6ZM0e233y5J6tixoxo0aKDevXtr69at2rx5s/r06aN27drphhtuKHT70dHR2rRpkw4ePKi//vpLubm5Gjx4sI4fP66ePXvq+++/1/79+7V06VI9+OCDysnJUVBQkPr376+RI0dq5cqV2rFjh/r16+fQRA6GYSg1NTXfozhHiaKjo7VixQqlpqbqxIkTRY7v1auXfv/9d33wwQfFnhjDEUOHDtWSJUv0yiuvaO/evXrnnXfsTiGUpLFjx+qTTz7R+PHjtXPnTu3evVuff/65nn32WUnS77//rkGDBunll1/WTTfdpBkzZuill17KF9oAAIUjbAEAHBYUFKSWLVvq9ddfV9u2bVW/fn0999xzGjBggN555x1JF05r++abb1ShQgW1bdtWHTt2VM2aNTVnzpwit//kk0/K29tbsbGxqly5sg4dOqTIyEitX79eOTk5io+PV4MGDTRs2DCFhobaAtXUqVPVpk0bde/eXR07dtRNN92kZs2aFfl+6enpqlq1ar6HI9dk5Xn11VeVmJio6tWrq0mTJkWODwkJUY8ePRQUFKQ77rjD4fdxVKtWrfTBBx/ozTffVKNGjbRs2TJbiMrTuXNnLVy4UMuWLVPz5s3VqlUrvf7664qKipJhGOrXr59atGihIUOG2MYPGjRI999/v06fPu30mgGgtLIYhmG4uwgAAMqSW265RfXq1dNbb73l7lIAAC5E2AIAoIScOHFCq1ev1t13361du3Y5NGMiAMBzMa0QAAAlpEmTJjpx4oRefvllghYAlAEc2QIAAAAAF2CCDAAAAABwAcIWAAAAALgAYQsAAAAAXICwBQAAAAAuQNgCAAAAABcgbAEAAACACxC2AAAAAMAFCFsAAAAA4AL/D2JnpGzt9EjJAAAAAElFTkSuQmCC\n" }, "metadata": {} } ], "source": [ "import matplotlib.pyplot as plt\n", "plt.figure(figsize=(10, 6))\n", "plt.plot(df['tok'].values, marker='o', linestyle='-')\n", "plt.xlabel('Sorted Entry Index')\n", "plt.ylabel('Token Count')\n", "plt.title(\"Line Plot of Token Counts (Sorted Ascending)\")\n", "plt.grid(True)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 13, "id": "01127bdd-aa73-4118-a5b8-53e5ed709bf5", "metadata": { "id": "01127bdd-aa73-4118-a5b8-53e5ed709bf5", "colab": { "base_uri": "https://localhost:8080/", "height": 363 }, "outputId": "7ac9fb40-3444-419c-c297-2d23bc9a56d3" }, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ " Input Output \\\n", "1262 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... B \n", "1202 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... A \n", "1289 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... A \n", "1246 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... D \n", "1358 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... D \n", "5938 यह प्रश्न निम्नलिखित जानकारी से संबंधित है।\\nस... A \n", "5874 यह प्रश्न निम्नलिखित जानकारी से संबंधित है।\\nस... B \n", "12115 साल 1993 में, एक पशुपालक किसान को 20 एकड़ के स... A \n", "10876 एक भूमि-विकास कंपनी के पास दक्षिणपश्चिम में 40... C \n", "11923 साल 1993 में, एक ज़मींदार के पास एक अंगूर के ब... C \n", "\n", " Sub-Domain Dataset tok \n", "1262 college_medicine MMLU 1917 \n", "1202 college_medicine MMLU 1894 \n", "1289 college_medicine MMLU 1892 \n", "1246 college_medicine MMLU 1831 \n", "1358 college_medicine MMLU 1830 \n", "5938 high_school_world_history MMLU 1313 \n", "5874 high_school_world_history MMLU 1292 \n", "12115 professional_law MMLU 1279 \n", "10876 professional_law MMLU 1277 \n", "11923 professional_law MMLU 1266 " ], "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
InputOutputSub-DomainDatasettok
1262सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Bcollege_medicineMMLU1917
1202सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Acollege_medicineMMLU1894
1289सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Acollege_medicineMMLU1892
1246सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Dcollege_medicineMMLU1831
1358सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Dcollege_medicineMMLU1830
5938यह प्रश्न निम्नलिखित जानकारी से संबंधित है।\\nस...Ahigh_school_world_historyMMLU1313
5874यह प्रश्न निम्नलिखित जानकारी से संबंधित है।\\nस...Bhigh_school_world_historyMMLU1292
12115साल 1993 में, एक पशुपालक किसान को 20 एकड़ के स...Aprofessional_lawMMLU1279
10876एक भूमि-विकास कंपनी के पास दक्षिणपश्चिम में 40...Cprofessional_lawMMLU1277
11923साल 1993 में, एक ज़मींदार के पास एक अंगूर के ब...Cprofessional_lawMMLU1266
\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", " \n", "\n", "\n", "\n", " \n", "
\n", "\n", "
\n", "
\n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "variable_name": "df", "summary": "{\n \"name\": \"df\",\n \"rows\": 14042,\n \"fields\": [\n {\n \"column\": \"Input\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 13981,\n \"samples\": [\n \"\\u0928\\u093f\\u092e\\u094d\\u0928\\u0932\\u093f\\u0916\\u093f\\u0924 \\u092e\\u0947\\u0902 \\u0938\\u0947 \\u0915\\u093f\\u0938 \\u092a\\u0926\\u093e\\u0930\\u094d\\u0925 \\u0915\\u0940 \\u0906\\u0923\\u094d\\u0935\\u093f\\u0915 \\u0938\\u0902\\u0930\\u091a\\u0928\\u093e \\u0905\\u0938\\u092e\\u092e\\u093f\\u0924 \\u0939\\u094b\\u0924\\u0940 \\u0939\\u0948? ### A) SF4 B) PCl5 C) BF3 D) CO2 ### MCQ ### \",\n \"\\u090f\\u0915\\u094d\\u0935\\u093f\\u0928\\u093e\\u0938 \\u0915\\u093e \\u0928\\u0948\\u0924\\u093f\\u0915 \\u0938\\u093f\\u0926\\u094d\\u0927\\u093e\\u0902\\u0924 \\u092a\\u094d\\u0930\\u093e\\u0915\\u0943\\u0924\\u093f\\u0915 \\u0915\\u093e\\u0928\\u0942\\u0928 \\u0938\\u093f\\u0926\\u094d\\u0927\\u093e\\u0902\\u0924 ### A) \\u092a\\u0930\\u093f\\u0923\\u093e\\u092e\\u0935\\u093e\\u0926\\u0964 B) \\u0915\\u093e \\u090f\\u0915 \\u0938\\u0902\\u0938\\u094d\\u0915\\u0930\\u0923 \\u0939\\u0948\\u0964 C) \\u0905\\u0927\\u093f\\u0915\\u093e\\u0930-\\u0906\\u0927\\u093e\\u0930\\u093f\\u0924 \\u0938\\u093f\\u0926\\u094d\\u0927\\u093e\\u0902\\u0924. D) \\u092a\\u0941\\u0923\\u094d\\u092f \\u0928\\u0948\\u0924\\u093f\\u0915\\u0924\\u093e\\u0964 ### MCQ ### \",\n \"\\u091c\\u093f\\u0938 \\u0926\\u0930 \\u092a\\u0930 \\u0936\\u0941\\u0926\\u094d\\u0927\\u093f\\u0915\\u0930\\u0923 \\u092a\\u094d\\u0930\\u0915\\u094d\\u0930\\u093f\\u092f\\u093e \\u092a\\u093e\\u0928\\u0940 \\u0915\\u0947 \\u091f\\u0948\\u0902\\u0915 \\u0938\\u0947 \\u0926\\u0942\\u0937\\u093f\\u0924 \\u092a\\u0926\\u093e\\u0930\\u094d\\u0925\\u094b\\u0902 \\u0915\\u094b \\u0939\\u091f\\u093e \\u0938\\u0915\\u0924\\u0940 \\u0939\\u0948, \\u0935\\u0939 \\u0936\\u0947\\u0937 \\u0926\\u0942\\u0937\\u093f\\u0924 \\u092a\\u0926\\u093e\\u0930\\u094d\\u0925\\u094b\\u0902 \\u0915\\u0940 \\u092e\\u093e\\u0924\\u094d\\u0930\\u093e \\u0915\\u0947 \\u0938\\u092e\\u093e\\u0928\\u0941\\u092a\\u093e\\u0924\\u0940 \\u0939\\u094b\\u0924\\u0940 \\u0939\\u0948. \\u0905\\u0917\\u0930 \\u092a\\u094d\\u0930\\u0915\\u094d\\u0930\\u093f\\u092f\\u093e \\u0915\\u0947 \\u092a\\u0939\\u0932\\u0947 \\u092e\\u093f\\u0928\\u091f \\u0915\\u0947 \\u0926\\u094c\\u0930\\u093e\\u0928 20% \\u0926\\u0942\\u0937\\u0915 \\u0915\\u094b \\u0939\\u091f\\u093e\\u092f\\u093e \\u091c\\u093e \\u0938\\u0915\\u0924\\u093e \\u0939\\u0948 \\u0914\\u0930 \\u092a\\u093e\\u0928\\u0940 \\u0915\\u094b \\u0938\\u0941\\u0930\\u0915\\u094d\\u0937\\u093f\\u0924 \\u092c\\u0928\\u093e\\u0928\\u0947 \\u0915\\u0947 \\u0932\\u093f\\u090f 98% \\u0915\\u094b \\u0939\\u091f\\u093e\\u092f\\u093e \\u091c\\u093e\\u0928\\u093e \\u091a\\u093e\\u0939\\u093f\\u090f, \\u0924\\u094b \\u0936\\u0941\\u0926\\u094d\\u0927\\u093f\\u0915\\u0930\\u0923 \\u092a\\u094d\\u0930\\u0915\\u094d\\u0930\\u093f\\u092f\\u093e \\u092e\\u0947\\u0902 \\u0932\\u0917\\u092d\\u0917 \\u0915\\u093f\\u0924\\u0928\\u093e \\u0938\\u092e\\u092f \\u0932\\u0917\\u0947\\u0917\\u093e? ### A) 2 \\u092e\\u093f\\u0928\\u091f B) 5 \\u092e\\u093f\\u0928\\u091f C) 18 \\u092e\\u093f\\u0928\\u091f D) 20 \\u092e\\u093f\\u0928\\u091f ### MCQ ### \"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Output\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 4,\n \"samples\": [\n \"A\",\n \"C\",\n \"B\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Sub-Domain\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 57,\n \"samples\": [\n \"college_medicine\",\n \"security_studies\",\n \"high_school_microeconomics\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Dataset\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 1,\n \"samples\": [\n \"MMLU\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"tok\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 164,\n \"min\": 29,\n \"max\": 1917,\n \"num_unique_values\": 831,\n \"samples\": [\n 251\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" } }, "metadata": {}, "execution_count": 13 } ], "source": [ "df.head(10)" ] }, { "cell_type": "code", "execution_count": 14, "id": "5af6bcac-3c98-421b-8f4b-854b57db8416", "metadata": { "id": "5af6bcac-3c98-421b-8f4b-854b57db8416", "colab": { "base_uri": "https://localhost:8080/", "height": 104 }, "outputId": "fa34640d-b8ec-4707-c7e7-97402c6791f9" }, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "'from tqdm import tqdm\\nimport torch\\ndef generate_responses_phi4_batched(df, tokenizer, model, batch_size=8):\\n responses = []\\n for i in tqdm(range(0, len(df), batch_size), desc=\"Processing Batches\"):\\n batch = df[\\'Input\\'][i:i + batch_size].tolist()\\n prompts = [f\"### INPUT : {text} ### MCQ ### OUTPUT :\" for text in batch]\\n messages = [{\"role\": \"user\", \"content\": prompt} for prompt in prompts]\\n inputs = [tokenizer.apply_chat_template([msg], tokenize=True, add_generation_prompt=True, return_tensors=\"pt\") for msg in messages]\\n max_length = max(input_tensor.size(1) for input_tensor in inputs)\\n padded_inputs = []\\n for input_tensor in inputs:\\n pad_size = max_length - input_tensor.size(1)\\n padded_tensor = torch.nn.functional.pad(input_tensor, (0, pad_size), value=tokenizer.pad_token_id)\\n padded_inputs.append(padded_tensor)\\n batch_inputs = torch.cat(padded_inputs, dim=0).to(\"cuda\")\\n outputs = model.generate(input_ids=batch_inputs, max_new_tokens=20, use_cache=True, temperature=0.1, min_p=0.1, pad_token_id=tokenizer.eos_token_id)\\n batch_responses = tokenizer.batch_decode(outputs, skip_special_tokens=True)\\n for response in batch_responses:\\n processed_response = response.split(\"### OUTPUT :\\nassistant\")[-1].strip()\\n responses.append(processed_response)\\n df[\\'Response\\'] = responses\\n return df'" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "string" } }, "metadata": {}, "execution_count": 14 } ], "source": [ "\"\"\"from tqdm import tqdm\n", "import torch\n", "def generate_responses_phi4_batched(df, tokenizer, model, batch_size=8):\n", " responses = []\n", " for i in tqdm(range(0, len(df), batch_size), desc=\"Processing Batches\"):\n", " batch = df['Input'][i:i + batch_size].tolist()\n", " prompts = [f\"### INPUT : {text} ### MCQ ### OUTPUT :\" for text in batch]\n", " messages = [{\"role\": \"user\", \"content\": prompt} for prompt in prompts]\n", " inputs = [tokenizer.apply_chat_template([msg], tokenize=True, add_generation_prompt=True, return_tensors=\"pt\") for msg in messages]\n", " max_length = max(input_tensor.size(1) for input_tensor in inputs)\n", " padded_inputs = []\n", " for input_tensor in inputs:\n", " pad_size = max_length - input_tensor.size(1)\n", " padded_tensor = torch.nn.functional.pad(input_tensor, (0, pad_size), value=tokenizer.pad_token_id)\n", " padded_inputs.append(padded_tensor)\n", " batch_inputs = torch.cat(padded_inputs, dim=0).to(\"cuda\")\n", " outputs = model.generate(input_ids=batch_inputs, max_new_tokens=20, use_cache=True, temperature=0.1, min_p=0.1, pad_token_id=tokenizer.eos_token_id)\n", " batch_responses = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n", " for response in batch_responses:\n", " processed_response = response.split(\"### OUTPUT :\\nassistant\")[-1].strip()\n", " responses.append(processed_response)\n", " df['Response'] = responses\n", " return df\"\"\"" ] }, { "cell_type": "code", "source": [ "df[14000:]" ], "metadata": { "id": "DnE5jnbPTd03", "colab": { "base_uri": "https://localhost:8080/", "height": 1000 }, "outputId": "50df6f49-7e39-4a19-d1e1-bf9bf9b6ddab" }, "id": "DnE5jnbPTd03", "execution_count": 15, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ " Input Output \\\n", "2399 100 का तीन पांचवा भाग क्या है? ### A) 3 B) 5 ... D \n", "2277 कौन सी संख्या 7 का गुणज है? ### A) 27 B) 48 C... C \n", "7890 तीन स्टूज में से कौन अन्य से संबंधित नहीं था? ... B \n", "2274 −8 • (−4) का गुणनफल ज्ञात कीजिए। ### A) −4 B)... C \n", "7667 लौवर संग्रहालय कहां है? ### A) पेरिस B) ल्योन... A \n", "7410 पेरेग्रन किस तरह का जानवर है? ### A) मूज़ B) ... C \n", "669 पानी का pH मान कितना है? ### A) 3.5 B) 7 C) 1... B \n", "3204 Python 3 में, निम्न में से कौन सा फ़्लोर डिवीज... B \n", "7421 आप एक युवा गाय को क्या कहते हैं? ### A) घोड़... D \n", "2107 किस संख्या का निरपेक्ष मान 5 से अधिक है? ### ... A \n", "38 परिमित क्षेत्र Z_7 के लिए जनरेटर पता करें. ###... C \n", "7529 आपका हॉलक्स क्या है? ### A) कान का लोब B) जीभ... D \n", "2126 एक दशक में कितने वर्ष होते हैं? ### A) 5 B) 1... B \n", "7722 इनमें से कौन मसाला नहीं है? ### A) डिल B) सौं... C \n", "1552 WPA किस प्रकार के एन्क्रिप्शन का उपयोग करता है... C \n", "13949 भक्ति का प्रायः क्या अनुवाद किया जाता है? ### ... C \n", "3142 निम्नलिखित में से कौन सा एक रेडियोधर्मी तत्व ह... C \n", "2112 Compute 22 / 2 + 9. ### A) 10 B) 11 C) 20 D) ... C \n", "5077 निम्नलिखित में से कौन द्वितीयक प्रबलक है? ### ... C \n", "7872 बेसबॉल में कितनी गेंदें वॉक करती हैं? ### A) ... D \n", "7737 इनमें से कौन सी फलियां नहीं हैं? ### A) फलिया... C \n", "2150 8 + 8 ÷ 2 + 2 = ### A) 4 B) 8 C) 10 D) 14 ###... D \n", "3161 निम्नलिखित में से किसे मेटलॉइड माना जाता है? #... C \n", "8041 इनमें से कौन सी मिर्च का प्रकार नहीं है? ### ... B \n", "77 वलय Z x Z की विशेषता पता करें. ### A) 0 B) 3 ... A \n", "7744 जिराफ़ की कितनी आँखें होती हैं? ### A) एक बार... B \n", "1766 नीले रंग का पूरक रंग है ### A) मैजेंटा B) पील... B \n", "3899 सरकार इसका उपयोग करके मुद्रास्फीति को मापती है... C \n", "7548 सर्वहारा वर्ग क्या है? ### A) बेघर B) राजपरिव... D \n", "7686 अरबी अंक '2' को कैसे लिखा जाता है? ### A) 2 B... A \n", "7971 आइस हॉकी खेल में कितने पीरियड होते हैं? ### A... C \n", "7681 इनमें से कौन सा शब्द सही वर्तनी वाला है? ### ... C \n", "7851 बैकगैमौन कितने खिलाड़ियों का खेल है? ### A) द... A \n", "8138 गेरी एडम्स किस संगठन के अध्यक्ष हैं? ### A) G... C \n", "1190 (1+i)^10 = ### A) 1 B) i C) 32 D) 32i ### MCQ... D \n", "7624 इनमें से कौन सा सामान्यतः ज्ञात निवेश खाता है?... C \n", "8005 निम्नलिखित में से कौन सा तत्व धातु है? ### A)... D \n", "2087 −4 + ( −3 )= ### A) −7 B) −1 C) 1 D) 7 ### M... A \n", "8009 माणिक्य किस रंग का होता है? ### A) लाल B) काल... A \n", "7772 इनमें से कौन सा शब्द क्रिया विशेषण है? ### A)... D \n", "7585 बल = द्रव्यमान = क्या? ### A) वेग B) दूरी C) ... C \n", "2983 एंटीमोनी का प्रतीक … है ### A) W B) Sb C) Fe ... B \n", "\n", " Sub-Domain Dataset tok \n", "2399 elementary_mathematics MMLU 38 \n", "2277 elementary_mathematics MMLU 38 \n", "7890 miscellaneous MMLU 38 \n", "2274 elementary_mathematics MMLU 38 \n", "7667 miscellaneous MMLU 38 \n", "7410 miscellaneous MMLU 38 \n", "669 clinical_knowledge MMLU 38 \n", "3204 high_school_computer_science MMLU 38 \n", "7421 miscellaneous MMLU 38 \n", "2107 elementary_mathematics MMLU 38 \n", "38 abstract_algebra MMLU 38 \n", "7529 miscellaneous MMLU 38 \n", "2126 elementary_mathematics MMLU 37 \n", "7722 miscellaneous MMLU 37 \n", "1552 computer_security MMLU 37 \n", "13949 world_religions MMLU 37 \n", "3142 high_school_chemistry MMLU 37 \n", "2112 elementary_mathematics MMLU 37 \n", "5077 high_school_psychology MMLU 37 \n", "7872 miscellaneous MMLU 37 \n", "7737 miscellaneous MMLU 37 \n", "2150 elementary_mathematics MMLU 36 \n", "3161 high_school_chemistry MMLU 36 \n", "8041 miscellaneous MMLU 36 \n", "77 abstract_algebra MMLU 36 \n", "7744 miscellaneous MMLU 36 \n", "1766 conceptual_physics MMLU 36 \n", "3899 high_school_macroeconomics MMLU 35 \n", "7548 miscellaneous MMLU 35 \n", "7686 miscellaneous MMLU 35 \n", "7971 miscellaneous MMLU 35 \n", "7681 miscellaneous MMLU 35 \n", "7851 miscellaneous MMLU 35 \n", "8138 miscellaneous MMLU 35 \n", "1190 college_mathematics MMLU 33 \n", "7624 miscellaneous MMLU 33 \n", "8005 miscellaneous MMLU 33 \n", "2087 elementary_mathematics MMLU 32 \n", "8009 miscellaneous MMLU 32 \n", "7772 miscellaneous MMLU 31 \n", "7585 miscellaneous MMLU 31 \n", "2983 high_school_chemistry MMLU 29 " ], "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
InputOutputSub-DomainDatasettok
2399100 का तीन पांचवा भाग क्या है? ### A) 3 B) 5 ...Delementary_mathematicsMMLU38
2277कौन सी संख्या 7 का गुणज है? ### A) 27 B) 48 C...Celementary_mathematicsMMLU38
7890तीन स्टूज में से कौन अन्य से संबंधित नहीं था? ...BmiscellaneousMMLU38
2274−8 • (−4) का गुणनफल ज्ञात कीजिए। ### A) −4 B)...Celementary_mathematicsMMLU38
7667लौवर संग्रहालय कहां है? ### A) पेरिस B) ल्योन...AmiscellaneousMMLU38
7410पेरेग्रन किस तरह का जानवर है? ### A) मूज़ B) ...CmiscellaneousMMLU38
669पानी का pH मान कितना है? ### A) 3.5 B) 7 C) 1...Bclinical_knowledgeMMLU38
3204Python 3 में, निम्न में से कौन सा फ़्लोर डिवीज...Bhigh_school_computer_scienceMMLU38
7421आप एक युवा गाय को क्या कहते हैं? ### A) घोड़...DmiscellaneousMMLU38
2107किस संख्या का निरपेक्ष मान 5 से अधिक है? ### ...Aelementary_mathematicsMMLU38
38परिमित क्षेत्र Z_7 के लिए जनरेटर पता करें. ###...Cabstract_algebraMMLU38
7529आपका हॉलक्स क्या है? ### A) कान का लोब B) जीभ...DmiscellaneousMMLU38
2126एक दशक में कितने वर्ष होते हैं? ### A) 5 B) 1...Belementary_mathematicsMMLU37
7722इनमें से कौन मसाला नहीं है? ### A) डिल B) सौं...CmiscellaneousMMLU37
1552WPA किस प्रकार के एन्क्रिप्शन का उपयोग करता है...Ccomputer_securityMMLU37
13949भक्ति का प्रायः क्या अनुवाद किया जाता है? ### ...Cworld_religionsMMLU37
3142निम्नलिखित में से कौन सा एक रेडियोधर्मी तत्व ह...Chigh_school_chemistryMMLU37
2112Compute 22 / 2 + 9. ### A) 10 B) 11 C) 20 D) ...Celementary_mathematicsMMLU37
5077निम्नलिखित में से कौन द्वितीयक प्रबलक है? ### ...Chigh_school_psychologyMMLU37
7872बेसबॉल में कितनी गेंदें वॉक करती हैं? ### A) ...DmiscellaneousMMLU37
7737इनमें से कौन सी फलियां नहीं हैं? ### A) फलिया...CmiscellaneousMMLU37
21508 + 8 ÷ 2 + 2 = ### A) 4 B) 8 C) 10 D) 14 ###...Delementary_mathematicsMMLU36
3161निम्नलिखित में से किसे मेटलॉइड माना जाता है? #...Chigh_school_chemistryMMLU36
8041इनमें से कौन सी मिर्च का प्रकार नहीं है? ### ...BmiscellaneousMMLU36
77वलय Z x Z की विशेषता पता करें. ### A) 0 B) 3 ...Aabstract_algebraMMLU36
7744जिराफ़ की कितनी आँखें होती हैं? ### A) एक बार...BmiscellaneousMMLU36
1766नीले रंग का पूरक रंग है ### A) मैजेंटा B) पील...Bconceptual_physicsMMLU36
3899सरकार इसका उपयोग करके मुद्रास्फीति को मापती है...Chigh_school_macroeconomicsMMLU35
7548सर्वहारा वर्ग क्या है? ### A) बेघर B) राजपरिव...DmiscellaneousMMLU35
7686अरबी अंक '2' को कैसे लिखा जाता है? ### A) 2 B...AmiscellaneousMMLU35
7971आइस हॉकी खेल में कितने पीरियड होते हैं? ### A...CmiscellaneousMMLU35
7681इनमें से कौन सा शब्द सही वर्तनी वाला है? ### ...CmiscellaneousMMLU35
7851बैकगैमौन कितने खिलाड़ियों का खेल है? ### A) द...AmiscellaneousMMLU35
8138गेरी एडम्स किस संगठन के अध्यक्ष हैं? ### A) G...CmiscellaneousMMLU35
1190(1+i)^10 = ### A) 1 B) i C) 32 D) 32i ### MCQ...Dcollege_mathematicsMMLU33
7624इनमें से कौन सा सामान्यतः ज्ञात निवेश खाता है?...CmiscellaneousMMLU33
8005निम्नलिखित में से कौन सा तत्व धातु है? ### A)...DmiscellaneousMMLU33
2087−4 + ( −3 )= ### A) −7 B) −1 C) 1 D) 7 ### M...Aelementary_mathematicsMMLU32
8009माणिक्य किस रंग का होता है? ### A) लाल B) काल...AmiscellaneousMMLU32
7772इनमें से कौन सा शब्द क्रिया विशेषण है? ### A)...DmiscellaneousMMLU31
7585बल = द्रव्यमान = क्या? ### A) वेग B) दूरी C) ...CmiscellaneousMMLU31
2983एंटीमोनी का प्रतीक … है ### A) W B) Sb C) Fe ...Bhigh_school_chemistryMMLU29
\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", " \n", "\n", "\n", "\n", " \n", "
\n", "\n", "
\n", "
\n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "summary": "{\n \"name\": \"df[14000:]\",\n \"rows\": 42,\n \"fields\": [\n {\n \"column\": \"Input\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 42,\n \"samples\": [\n \"\\u091c\\u093f\\u0930\\u093e\\u092b\\u093c \\u0915\\u0940 \\u0915\\u093f\\u0924\\u0928\\u0940 \\u0906\\u0901\\u0916\\u0947\\u0902 \\u0939\\u094b\\u0924\\u0940 \\u0939\\u0948\\u0902? ### A) \\u090f\\u0915 \\u092c\\u093e\\u0930 B) \\u0926\\u094b \\u092c\\u093e\\u0930 C) \\u0924\\u0940\\u0928 \\u092c\\u093e\\u0930 D) \\u091a\\u093e\\u0930 ### MCQ ### \",\n \"\\u0907\\u0928\\u092e\\u0947\\u0902 \\u0938\\u0947 \\u0915\\u094c\\u0928 \\u092e\\u0938\\u093e\\u0932\\u093e \\u0928\\u0939\\u0940\\u0902 \\u0939\\u0948? ### A) \\u0921\\u093f\\u0932 B) \\u0938\\u094c\\u0902\\u092b C) \\u0916\\u0940\\u0930\\u093e D) \\u0915\\u0947\\u092f\\u0947\\u0928 ### MCQ ### \",\n \"\\u0906\\u092a \\u090f\\u0915 \\u092f\\u0941\\u0935\\u093e \\u0917\\u093e\\u092f \\u0915\\u094b \\u0915\\u094d\\u092f\\u093e \\u0915\\u0939\\u0924\\u0947 \\u0939\\u0948\\u0902? ### A) \\u0918\\u094b\\u0921\\u093c\\u0940 B) \\u092d\\u0947\\u0921\\u093c C) \\u092c\\u091a\\u094d\\u091a\\u093e D) \\u092c\\u091b\\u0921\\u093c\\u093e ### MCQ ### \"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Output\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 4,\n \"samples\": [\n \"C\",\n \"A\",\n \"D\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Sub-Domain\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 12,\n \"samples\": [\n \"high_school_macroeconomics\",\n \"conceptual_physics\",\n \"elementary_mathematics\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Dataset\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 1,\n \"samples\": [\n \"MMLU\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"tok\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 2,\n \"min\": 29,\n \"max\": 38,\n \"num_unique_values\": 8,\n \"samples\": [\n 37\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" } }, "metadata": {}, "execution_count": 15 } ] }, { "cell_type": "code", "source": [ "import torch\n", "import torch.nn.functional as F\n", "from transformers import AutoTokenizer, AutoModelForCausalLM\n", "input_text = \"जोड़ें 46,911 + 653,092 ### A) 699,903 B) 700,003 C) 913,203 D) 1,122,202 ### MCQ ###\tRespond with just one letter based on these options : \"\n", "prompt = f\"### INPUT : {input_text} RESPONSE : \"\n", "message = [{\"role\": \"user\", \"content\": prompt}]\n", "inputs = tokenizer.apply_chat_template(message, tokenize=True, add_generation_prompt=True, return_tensors=\"pt\").to(\"cuda\")\n", "with torch.no_grad():\n", " outputs = model.generate(input_ids=inputs, max_new_tokens=1, use_cache=True, pad_token_id=tokenizer.eos_token_id, return_dict_in_generate=True, output_scores=True)\n", " next_token_logits = outputs.scores[0]\n", "probs = F.softmax(next_token_logits, dim=-1)\n", "token_ids_a = tokenizer.encode('A', add_special_tokens=False)[0]\n", "token_ids_b = tokenizer.encode('B', add_special_tokens=False)[0]\n", "token_ids_c = tokenizer.encode('C', add_special_tokens=False)[0]\n", "token_ids_d = tokenizer.encode('D', add_special_tokens=False)[0]\n", "prob_a = probs[0, token_ids_a].item()\n", "prob_b = probs[0, token_ids_b].item()\n", "prob_c = probs[0, token_ids_c].item()\n", "prob_d = probs[0, token_ids_d].item()\n", "print(f\"Probability of 'A': {prob_a:.4f}\")\n", "print(f\"Probability of 'B': {prob_b:.4f}\")\n", "print(f\"Probability of 'C': {prob_c:.4f}\")\n", "print(f\"Probability of 'D': {prob_d:.4f}\")" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "x9Gu8g6zZy_Q", "outputId": "40fad9ca-c27b-4263-eb1a-ce6441ed113b" }, "id": "x9Gu8g6zZy_Q", "execution_count": 16, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n" ] }, { "output_type": "stream", "name": "stdout", "text": [ "Probability of 'A': 0.2380\n", "Probability of 'B': 0.3463\n", "Probability of 'C': 0.0992\n", "Probability of 'D': 0.0682\n" ] } ] }, { "cell_type": "code", "source": [ "import torch\n", "import torch.nn.functional as F\n", "from transformers import AutoTokenizer, AutoModelForCausalLM\n", "input_text = \"दिए गए बहुपद के संकेतित परिमित क्षेत्र में गुणांकों के साथ सभी शून्य बताएं. x^5 + 3x^3 + x^2 + 2x in Z_5 ### A) 0 B) 1 C) 0,1 D) 0,4\t### MCQ ### Respond with just one letter based on these options : \"\n", "prompt = f\"### INPUT : {input_text} RESPONSE : \"\n", "message = [{\"role\": \"user\", \"content\": prompt}]\n", "inputs = tokenizer.apply_chat_template(message, tokenize=True, add_generation_prompt=True, return_tensors=\"pt\").to(\"cuda\")\n", "outputs = model.generate(input_ids=inputs, max_new_tokens=200, use_cache=True, temperature=0.1, min_p=0.1, pad_token_id=tokenizer.eos_token_id)\n", "response = tokenizer.decode(outputs[0], skip_special_tokens=True)\n", "processed_response = response.split(\"### RESPONSE :\\nmodel\")[-1].strip()\n", "print(f\"Generated Response (20 tokens):\\n{processed_response}\\n\")\n", "with torch.no_grad():\n", " outputs = model.generate(input_ids=inputs, max_new_tokens=1, use_cache=True, pad_token_id=tokenizer.eos_token_id, return_dict_in_generate=True, output_scores=True)\n", " next_token_logits = outputs.scores[0]\n", "probs = F.softmax(next_token_logits, dim=-1)\n", "token_ids_a = tokenizer.encode('A', add_special_tokens=False)[0]\n", "token_ids_b = tokenizer.encode('B', add_special_tokens=False)[0]\n", "token_ids_c = tokenizer.encode('C', add_special_tokens=False)[0]\n", "token_ids_d = tokenizer.encode('D', add_special_tokens=False)[0]\n", "prob_a = probs[0, token_ids_a].item()\n", "prob_b = probs[0, token_ids_b].item()\n", "prob_c = probs[0, token_ids_c].item()\n", "prob_d = probs[0, token_ids_d].item()\n", "print(f\"Probability of 'A': {prob_a:.4f}\")\n", "print(f\"Probability of 'B': {prob_b:.4f}\")\n", "print(f\"Probability of 'C': {prob_c:.4f}\")\n", "print(f\"Probability of 'D': {prob_d:.4f}\")" ], "metadata": { "id": "QuuDF-qci19H", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "bb16238f-db08-45b5-b5e8-368f720eee39" }, "id": "QuuDF-qci19H", "execution_count": 17, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Generated Response (20 tokens):\n", "user\n", "### INPUT : दिए गए बहुपद के संकेतित परिमित क्षेत्र में गुणांकों के साथ सभी शून्य बताएं. x^5 + 3x^3 + x^2 + 2x in Z_5 ### A) 0 B) 1 C) 0,1 D) 0,4\t### MCQ ### Respond with just one letter based on these options : RESPONSE :\n", "model\n", "The correct answer is C. Here's why:\n", "\n", "**Understanding the problem:**\n", "\n", "We are given a polynomial with coefficients in the finite field Z_5 (the integers modulo 5). We need to find all the roots (or zeros) of this polynomial within Z_5.\n", "\n", "**Solving the problem:**\n", "\n", "1. **Substitute values:** Since we are working in Z_5, we can substitute the elements of Z_5 (0, 1, 2, 3, 4) into the polynomial to see if they result in 0.\n", "\n", "2. **Check for roots:**\n", " * x = 0: 0^5 + 3(0)^3 + 0^2 + 2(0) = 0 \n", " * x = 1: 1^5 + 3(1)^3 + 1^2 + 2(1) = 1 + 3 + 1 + 2\n", "\n", "Probability of 'A': 0.0537\n", "Probability of 'B': 0.0781\n", "Probability of 'C': 0.1288\n", "Probability of 'D': 0.0690\n" ] } ] }, { "cell_type": "code", "source": [ "import torch\n", "import torch.nn.functional as F\n", "input_text = \"जोड़ें 46,911 + 653,092 ### A) 699,903 B) 700,003 C) 913,203 D) 1,122,202 ### MCQ ###\"\n", "prompt = f\"### INPUT : {input_text} RESPONSE : \"\n", "message = [{\"role\": \"user\", \"content\": prompt}]\n", "inputs = tokenizer.apply_chat_template(message, tokenize=True, add_generation_prompt=True, return_tensors=\"pt\").to(\"cuda\")\n", "outputs = model.generate(input_ids=inputs, max_new_tokens=200, use_cache=True, temperature=0.1, min_p=0.1, pad_token_id=tokenizer.eos_token_id)\n", "response = tokenizer.decode(outputs[0], skip_special_tokens=True)\n", "processed_response = response.split(\"### RESPONSE :\\nmodel\")[-1].strip()\n", "print(f\"Generated Response (20 tokens):\\n{processed_response}\\n\")\n", "with torch.no_grad():\n", " outputs = model.generate(input_ids=inputs, max_new_tokens=3, use_cache=True, pad_token_id=tokenizer.eos_token_id, return_dict_in_generate=True, output_scores=True)\n", " scores = outputs.scores\n", "token_ids_a = tokenizer.encode('A', add_special_tokens=False)[0]\n", "token_ids_b = tokenizer.encode('B', add_special_tokens=False)[0]\n", "token_ids_c = tokenizer.encode('C', add_special_tokens=False)[0]\n", "token_ids_d = tokenizer.encode('D', add_special_tokens=False)[0]\n", "for i, score in enumerate(scores, 1):\n", " probs = F.softmax(score, dim=-1)\n", " prob_a = probs[0, token_ids_a].item()\n", " prob_b = probs[0, token_ids_b].item()\n", " prob_c = probs[0, token_ids_c].item()\n", " prob_d = probs[0, token_ids_d].item()\n", " print(f\"Probability of 'A' at token {i}: {prob_a:.4f}\")\n", " print(f\"Probability of 'B' at token {i}: {prob_b:.4f}\")\n", " print(f\"Probability of 'C' at token {i}: {prob_c:.4f}\")\n", " print(f\"Probability of 'D' at token {i}: {prob_d:.4f}\")" ], "metadata": { "id": "r1dozae-gO5B", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "a53ffee0-d41f-4984-b3ad-ea43fe900872" }, "id": "r1dozae-gO5B", "execution_count": 18, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Generated Response (20 tokens):\n", "The correct answer is **B) 700,003**. \n", "\n", "Here's how to add the numbers:\n", "\n", "```\n", " 46,911\n", "+ 653,092\n", "----------\n", " 700,003 \n", "```\n", "\n", "Probability of 'A' at token 1: 0.0103\n", "Probability of 'B' at token 1: 0.0038\n", "Probability of 'C' at token 1: 0.0010\n", "Probability of 'D' at token 1: 0.0020\n", "Probability of 'A' at token 2: 0.0000\n", "Probability of 'B' at token 2: 0.0000\n", "Probability of 'C' at token 2: 0.0000\n", "Probability of 'D' at token 2: 0.0000\n", "Probability of 'A' at token 3: 0.0000\n", "Probability of 'B' at token 3: 0.0000\n", "Probability of 'C' at token 3: 0.0000\n", "Probability of 'D' at token 3: 0.0000\n" ] } ] }, { "cell_type": "code", "source": [ "import torch\n", "import torch.nn.functional as F\n", "from transformers import AutoTokenizer, AutoModelForCausalLM\n", "from tqdm import tqdm\n", "responses = []\n", "prob_a1_list = []\n", "prob_a2_list = []\n", "prob_a3_list = []\n", "prob_b1_list = []\n", "prob_b2_list = []\n", "prob_b3_list = []\n", "prob_c1_list = []\n", "prob_c2_list = []\n", "prob_c3_list = []\n", "prob_d1_list = []\n", "prob_d2_list = []\n", "prob_d3_list = []\n", "batch_size = 1\n", "for start in tqdm(range(0, len(df), batch_size)):\n", " batch_texts = df['Input'][start:start+batch_size].tolist()\n", " for input_text in batch_texts:\n", " prompt = f\"### INPUT : {input_text} Respond with just one letter based on these options : \"\n", " message = [{\"role\": \"user\", \"content\": prompt}]\n", " inputs = tokenizer.apply_chat_template(message, tokenize=True, add_generation_prompt=True, return_tensors=\"pt\").to(\"cuda\")\n", " with torch.no_grad():\n", " outputs = model.generate(input_ids=inputs, max_new_tokens=3, use_cache=True, pad_token_id=tokenizer.eos_token_id, return_dict_in_generate=True, output_scores=True)\n", " scores = outputs.scores # tuple of [batch_size, vocab_size] for each token\n", " token_ids_a = tokenizer.encode('A', add_special_tokens=False)[0]\n", " token_ids_b = tokenizer.encode('B', add_special_tokens=False)[0]\n", " token_ids_c = tokenizer.encode('C', add_special_tokens=False)[0]\n", " token_ids_d = tokenizer.encode('D', add_special_tokens=False)[0]\n", " for i in range(3):\n", " if i < len(scores):\n", " probs = F.softmax(scores[i], dim=-1)\n", " prob_a = probs[0, token_ids_a].item()\n", " prob_b = probs[0, token_ids_b].item()\n", " prob_c = probs[0, token_ids_c].item()\n", " prob_d = probs[0, token_ids_d].item()\n", " else:\n", " prob_a, prob_b, prob_c, prob_d = 0.0, 0.0, 0.0, 0.0\n", " if i == 0:\n", " prob_a1_list.append(prob_a)\n", " prob_b1_list.append(prob_b)\n", " prob_c1_list.append(prob_c)\n", " prob_d1_list.append(prob_d)\n", " elif i == 1:\n", " prob_a2_list.append(prob_a)\n", " prob_b2_list.append(prob_b)\n", " prob_c2_list.append(prob_c)\n", " prob_d2_list.append(prob_d)\n", " elif i == 2:\n", " prob_a3_list.append(prob_a)\n", " prob_b3_list.append(prob_b)\n", " prob_c3_list.append(prob_c)\n", " prob_d3_list.append(prob_d)\n", "df['A1'] = prob_a1_list\n", "df['A2'] = prob_a2_list\n", "df['A3'] = prob_a3_list\n", "df['B1'] = prob_b1_list\n", "df['B2'] = prob_b2_list\n", "df['B3'] = prob_b3_list\n", "df['C1'] = prob_c1_list\n", "df['C2'] = prob_c2_list\n", "df['C3'] = prob_c3_list\n", "df['D1'] = prob_d1_list\n", "df['D2'] = prob_d2_list\n", "df['D3'] = prob_d3_list" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "uyYEgG7_khiV", "outputId": "4b8face6-0e6e-4f43-96fa-93f94e767213" }, "id": "uyYEgG7_khiV", "execution_count": 19, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "100%|██████████| 14042/14042 [52:49<00:00, 4.43it/s]\n" ] } ] }, { "cell_type": "code", "source": [ "\"\"\"import torch\n", "import torch.nn.functional as F\n", "from transformers import AutoTokenizer, AutoModelForCausalLM\n", "responses = []\n", "prob_a_list = []\n", "prob_b_list = []\n", "prob_c_list = []\n", "prob_d_list = []\n", "batch_size = 1\n", "for start in tqdm(range(0, len(df), batch_size)):\n", " batch_texts = df['Input'][start:start+batch_size].tolist()\n", " for input_text in batch_texts:\n", " prompt = f\"### INPUT : {input_text} Respond with just one letter based on these options : \"\n", " message = [{\"role\": \"user\", \"content\": prompt}]\n", " inputs = tokenizer.apply_chat_template(message, tokenize=True, add_generation_prompt=True, return_tensors=\"pt\").to(\"cuda\")\n", " with torch.no_grad():\n", " outputs = model.generate(input_ids=inputs, max_new_tokens=1, use_cache=True, pad_token_id=tokenizer.eos_token_id, return_dict_in_generate=True, output_scores=True)\n", " next_token_logits = outputs.scores[0]\n", " probs = F.softmax(next_token_logits, dim=-1)\n", " token_ids_a = tokenizer.encode('A', add_special_tokens=False)[0]\n", " token_ids_b = tokenizer.encode('B', add_special_tokens=False)[0]\n", " token_ids_c = tokenizer.encode('C', add_special_tokens=False)[0]\n", " token_ids_d = tokenizer.encode('D', add_special_tokens=False)[0]\n", " prob_a_list.append(probs[0, token_ids_a].item())\n", " prob_b_list.append(probs[0, token_ids_b].item())\n", " prob_c_list.append(probs[0, token_ids_c].item())\n", " prob_d_list.append(probs[0, token_ids_d].item())\n", "df['A'] = prob_a_list\n", "df['B'] = prob_b_list\n", "df['C'] = prob_c_list\n", "df['D'] = prob_d_list\"\"\"" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 104 }, "id": "WFl4ieGwkF0G", "outputId": "325a9463-c286-445f-ed06-45e8c52cbe05" }, "id": "WFl4ieGwkF0G", "execution_count": 20, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "'import torch\\nimport torch.nn.functional as F\\nfrom transformers import AutoTokenizer, AutoModelForCausalLM\\nresponses = []\\nprob_a_list = []\\nprob_b_list = []\\nprob_c_list = []\\nprob_d_list = []\\nbatch_size = 1\\nfor start in tqdm(range(0, len(df), batch_size)):\\n batch_texts = df[\\'Input\\'][start:start+batch_size].tolist()\\n for input_text in batch_texts:\\n prompt = f\"### INPUT : {input_text} Respond with just one letter based on these options : \"\\n message = [{\"role\": \"user\", \"content\": prompt}]\\n inputs = tokenizer.apply_chat_template(message, tokenize=True, add_generation_prompt=True, return_tensors=\"pt\").to(\"cuda\")\\n with torch.no_grad():\\n outputs = model.generate(input_ids=inputs, max_new_tokens=1, use_cache=True, pad_token_id=tokenizer.eos_token_id, return_dict_in_generate=True, output_scores=True)\\n next_token_logits = outputs.scores[0]\\n probs = F.softmax(next_token_logits, dim=-1)\\n token_ids_a = tokenizer.encode(\\'A\\', add_special_tokens=False)[0]\\n token_ids_b = tokenizer.encode(\\'B\\', add_special_tokens=False)[0]\\n token_ids_c = tokenizer.encode(\\'C\\', add_special_tokens=False)[0]\\n token_ids_d = tokenizer.encode(\\'D\\', add_special_tokens=False)[0]\\n prob_a_list.append(probs[0, token_ids_a].item())\\n prob_b_list.append(probs[0, token_ids_b].item())\\n prob_c_list.append(probs[0, token_ids_c].item())\\n prob_d_list.append(probs[0, token_ids_d].item())\\ndf[\\'A\\'] = prob_a_list\\ndf[\\'B\\'] = prob_b_list\\ndf[\\'C\\'] = prob_c_list\\ndf[\\'D\\'] = prob_d_list'" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "string" } }, "metadata": {}, "execution_count": 20 } ] }, { "cell_type": "code", "source": [ "df" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 423 }, "id": "y9FMYJ1s2BZG", "outputId": "bfb6a921-4920-4d28-ee8a-c77ffa499a52" }, "id": "y9FMYJ1s2BZG", "execution_count": 21, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ " Input Output \\\n", "1262 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... B \n", "1202 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... A \n", "1289 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... A \n", "1246 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... D \n", "1358 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... D \n", "... ... ... \n", "2087 −4 + ( −3 )= ### A) −7 B) −1 C) 1 D) 7 ### M... A \n", "8009 माणिक्य किस रंग का होता है? ### A) लाल B) काल... A \n", "7772 इनमें से कौन सा शब्द क्रिया विशेषण है? ### A)... D \n", "7585 बल = द्रव्यमान = क्या? ### A) वेग B) दूरी C) ... C \n", "2983 एंटीमोनी का प्रतीक … है ### A) W B) Sb C) Fe ... B \n", "\n", " Sub-Domain Dataset tok A1 A2 A3 \\\n", "1262 college_medicine MMLU 1917 0.616334 3.182060e-07 0.0 \n", "1202 college_medicine MMLU 1894 0.963845 1.845272e-07 0.0 \n", "1289 college_medicine MMLU 1892 0.523709 2.649202e-07 0.0 \n", "1246 college_medicine MMLU 1831 0.205032 8.206729e-08 0.0 \n", "1358 college_medicine MMLU 1830 0.017179 3.866005e-08 0.0 \n", "... ... ... ... ... ... ... \n", "2087 elementary_mathematics MMLU 32 0.894250 9.286754e-08 0.0 \n", "8009 miscellaneous MMLU 32 0.975530 9.412071e-07 0.0 \n", "7772 miscellaneous MMLU 31 0.065354 1.627498e-07 0.0 \n", "7585 miscellaneous MMLU 31 0.089666 1.529922e-07 0.0 \n", "2983 high_school_chemistry MMLU 29 0.031167 1.603878e-07 0.0 \n", "\n", " B1 B2 B3 C1 C2 C3 D1 \\\n", "1262 0.291135 6.803543e-06 0.0 0.027080 1.503099e-07 0.0 0.050592 \n", "1202 0.022667 5.339471e-07 0.0 0.005731 5.286792e-08 0.0 0.001642 \n", "1289 0.132414 2.675599e-06 0.0 0.015815 1.820768e-07 0.0 0.317646 \n", "1246 0.715631 1.632103e-07 0.0 0.024488 2.864430e-07 0.0 0.040373 \n", "1358 0.937922 9.774791e-09 0.0 0.011807 3.866005e-08 0.0 0.024995 \n", "... ... ... ... ... ... ... ... \n", "2087 0.050450 7.775704e-07 0.0 0.018560 2.660700e-08 0.0 0.016379 \n", "8009 0.010837 2.533224e-07 0.0 0.004518 6.404995e-08 0.0 0.001883 \n", "7772 0.014582 2.344633e-08 0.0 0.007805 1.335932e-08 0.0 0.902178 \n", "7585 0.025690 2.858271e-07 0.0 0.850730 8.717234e-08 0.0 0.015582 \n", "2983 0.910824 4.640974e-07 0.0 0.003722 5.598083e-07 0.0 0.005416 \n", "\n", " D2 D3 \n", "1262 4.879847e-08 0.0 \n", "1202 3.275749e-09 0.0 \n", "1289 4.603623e-08 0.0 \n", "1246 4.928523e-09 0.0 \n", "1358 1.499013e-09 0.0 \n", "... ... ... \n", "2087 1.613796e-08 0.0 \n", "8009 2.669999e-08 0.0 \n", "7772 5.928161e-09 0.0 \n", "7585 8.717234e-08 0.0 \n", "2983 2.787121e-08 0.0 \n", "\n", "[14042 rows x 17 columns]" ], "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
InputOutputSub-DomainDatasettokA1A2A3B1B2B3C1C2C3D1D2D3
1262सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Bcollege_medicineMMLU19170.6163343.182060e-070.00.2911356.803543e-060.00.0270801.503099e-070.00.0505924.879847e-080.0
1202सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Acollege_medicineMMLU18940.9638451.845272e-070.00.0226675.339471e-070.00.0057315.286792e-080.00.0016423.275749e-090.0
1289सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Acollege_medicineMMLU18920.5237092.649202e-070.00.1324142.675599e-060.00.0158151.820768e-070.00.3176464.603623e-080.0
1246सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Dcollege_medicineMMLU18310.2050328.206729e-080.00.7156311.632103e-070.00.0244882.864430e-070.00.0403734.928523e-090.0
1358सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Dcollege_medicineMMLU18300.0171793.866005e-080.00.9379229.774791e-090.00.0118073.866005e-080.00.0249951.499013e-090.0
......................................................
2087−4 + ( −3 )= ### A) −7 B) −1 C) 1 D) 7 ### M...Aelementary_mathematicsMMLU320.8942509.286754e-080.00.0504507.775704e-070.00.0185602.660700e-080.00.0163791.613796e-080.0
8009माणिक्य किस रंग का होता है? ### A) लाल B) काल...AmiscellaneousMMLU320.9755309.412071e-070.00.0108372.533224e-070.00.0045186.404995e-080.00.0018832.669999e-080.0
7772इनमें से कौन सा शब्द क्रिया विशेषण है? ### A)...DmiscellaneousMMLU310.0653541.627498e-070.00.0145822.344633e-080.00.0078051.335932e-080.00.9021785.928161e-090.0
7585बल = द्रव्यमान = क्या? ### A) वेग B) दूरी C) ...CmiscellaneousMMLU310.0896661.529922e-070.00.0256902.858271e-070.00.8507308.717234e-080.00.0155828.717234e-080.0
2983एंटीमोनी का प्रतीक … है ### A) W B) Sb C) Fe ...Bhigh_school_chemistryMMLU290.0311671.603878e-070.00.9108244.640974e-070.00.0037225.598083e-070.00.0054162.787121e-080.0
\n", "

14042 rows × 17 columns

\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", " \n", "\n", "\n", "\n", " \n", "
\n", "\n", "
\n", " \n", " \n", " \n", "
\n", "\n", "
\n", "
\n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "variable_name": "df", "summary": "{\n \"name\": \"df\",\n \"rows\": 14042,\n \"fields\": [\n {\n \"column\": \"Input\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 13981,\n \"samples\": [\n \"\\u0928\\u093f\\u092e\\u094d\\u0928\\u0932\\u093f\\u0916\\u093f\\u0924 \\u092e\\u0947\\u0902 \\u0938\\u0947 \\u0915\\u093f\\u0938 \\u092a\\u0926\\u093e\\u0930\\u094d\\u0925 \\u0915\\u0940 \\u0906\\u0923\\u094d\\u0935\\u093f\\u0915 \\u0938\\u0902\\u0930\\u091a\\u0928\\u093e \\u0905\\u0938\\u092e\\u092e\\u093f\\u0924 \\u0939\\u094b\\u0924\\u0940 \\u0939\\u0948? ### A) SF4 B) PCl5 C) BF3 D) CO2 ### MCQ ### \",\n \"\\u090f\\u0915\\u094d\\u0935\\u093f\\u0928\\u093e\\u0938 \\u0915\\u093e \\u0928\\u0948\\u0924\\u093f\\u0915 \\u0938\\u093f\\u0926\\u094d\\u0927\\u093e\\u0902\\u0924 \\u092a\\u094d\\u0930\\u093e\\u0915\\u0943\\u0924\\u093f\\u0915 \\u0915\\u093e\\u0928\\u0942\\u0928 \\u0938\\u093f\\u0926\\u094d\\u0927\\u093e\\u0902\\u0924 ### A) \\u092a\\u0930\\u093f\\u0923\\u093e\\u092e\\u0935\\u093e\\u0926\\u0964 B) \\u0915\\u093e \\u090f\\u0915 \\u0938\\u0902\\u0938\\u094d\\u0915\\u0930\\u0923 \\u0939\\u0948\\u0964 C) \\u0905\\u0927\\u093f\\u0915\\u093e\\u0930-\\u0906\\u0927\\u093e\\u0930\\u093f\\u0924 \\u0938\\u093f\\u0926\\u094d\\u0927\\u093e\\u0902\\u0924. D) \\u092a\\u0941\\u0923\\u094d\\u092f \\u0928\\u0948\\u0924\\u093f\\u0915\\u0924\\u093e\\u0964 ### MCQ ### \",\n \"\\u091c\\u093f\\u0938 \\u0926\\u0930 \\u092a\\u0930 \\u0936\\u0941\\u0926\\u094d\\u0927\\u093f\\u0915\\u0930\\u0923 \\u092a\\u094d\\u0930\\u0915\\u094d\\u0930\\u093f\\u092f\\u093e \\u092a\\u093e\\u0928\\u0940 \\u0915\\u0947 \\u091f\\u0948\\u0902\\u0915 \\u0938\\u0947 \\u0926\\u0942\\u0937\\u093f\\u0924 \\u092a\\u0926\\u093e\\u0930\\u094d\\u0925\\u094b\\u0902 \\u0915\\u094b \\u0939\\u091f\\u093e \\u0938\\u0915\\u0924\\u0940 \\u0939\\u0948, \\u0935\\u0939 \\u0936\\u0947\\u0937 \\u0926\\u0942\\u0937\\u093f\\u0924 \\u092a\\u0926\\u093e\\u0930\\u094d\\u0925\\u094b\\u0902 \\u0915\\u0940 \\u092e\\u093e\\u0924\\u094d\\u0930\\u093e \\u0915\\u0947 \\u0938\\u092e\\u093e\\u0928\\u0941\\u092a\\u093e\\u0924\\u0940 \\u0939\\u094b\\u0924\\u0940 \\u0939\\u0948. \\u0905\\u0917\\u0930 \\u092a\\u094d\\u0930\\u0915\\u094d\\u0930\\u093f\\u092f\\u093e \\u0915\\u0947 \\u092a\\u0939\\u0932\\u0947 \\u092e\\u093f\\u0928\\u091f \\u0915\\u0947 \\u0926\\u094c\\u0930\\u093e\\u0928 20% \\u0926\\u0942\\u0937\\u0915 \\u0915\\u094b \\u0939\\u091f\\u093e\\u092f\\u093e \\u091c\\u093e \\u0938\\u0915\\u0924\\u093e \\u0939\\u0948 \\u0914\\u0930 \\u092a\\u093e\\u0928\\u0940 \\u0915\\u094b \\u0938\\u0941\\u0930\\u0915\\u094d\\u0937\\u093f\\u0924 \\u092c\\u0928\\u093e\\u0928\\u0947 \\u0915\\u0947 \\u0932\\u093f\\u090f 98% \\u0915\\u094b \\u0939\\u091f\\u093e\\u092f\\u093e \\u091c\\u093e\\u0928\\u093e \\u091a\\u093e\\u0939\\u093f\\u090f, \\u0924\\u094b \\u0936\\u0941\\u0926\\u094d\\u0927\\u093f\\u0915\\u0930\\u0923 \\u092a\\u094d\\u0930\\u0915\\u094d\\u0930\\u093f\\u092f\\u093e \\u092e\\u0947\\u0902 \\u0932\\u0917\\u092d\\u0917 \\u0915\\u093f\\u0924\\u0928\\u093e \\u0938\\u092e\\u092f \\u0932\\u0917\\u0947\\u0917\\u093e? ### A) 2 \\u092e\\u093f\\u0928\\u091f B) 5 \\u092e\\u093f\\u0928\\u091f C) 18 \\u092e\\u093f\\u0928\\u091f D) 20 \\u092e\\u093f\\u0928\\u091f ### MCQ ### \"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Output\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 4,\n \"samples\": [\n \"A\",\n \"C\",\n \"B\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Sub-Domain\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 57,\n \"samples\": [\n \"college_medicine\",\n \"security_studies\",\n \"high_school_microeconomics\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Dataset\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 1,\n \"samples\": [\n \"MMLU\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"tok\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 164,\n \"min\": 29,\n \"max\": 1917,\n \"num_unique_values\": 831,\n \"samples\": [\n 251\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"A1\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.2295292572210592,\n \"min\": 0.004024200141429901,\n \"max\": 0.9924488067626953,\n \"num_unique_values\": 13978,\n \"samples\": [\n 0.8465485572814941\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"A2\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 6.662313030298556e-07,\n \"min\": 8.773767823377909e-11,\n \"max\": 1.955147490662057e-05,\n \"num_unique_values\": 13960,\n \"samples\": [\n 4.3750741696158e-08\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"A3\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 8.015741729340901e-11,\n \"min\": 0.0,\n \"max\": 6.944549824083879e-09,\n \"num_unique_values\": 4,\n \"samples\": [\n 5.7928453145450476e-09\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"B1\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.23698803171122576,\n \"min\": 0.000481221271911636,\n \"max\": 0.9846892356872559,\n \"num_unique_values\": 13979,\n \"samples\": [\n 0.05411803722381592\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"B2\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 6.875979017109163e-07,\n \"min\": 3.02727648504586e-10,\n \"max\": 2.4851480702636763e-05,\n \"num_unique_values\": 13964,\n \"samples\": [\n 9.308511295103017e-08\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"B3\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 2.5427805937810923e-11,\n \"min\": 0.0,\n \"max\": 2.126724574225136e-09,\n \"num_unique_values\": 4,\n \"samples\": [\n 1.2142463701181327e-09\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"C1\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.23604838627936287,\n \"min\": 9.475808474235237e-05,\n \"max\": 0.984950065612793,\n \"num_unique_values\": 13978,\n \"samples\": [\n 0.03719474747776985\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"C2\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 6.286397050712039e-07,\n \"min\": 6.885907816167602e-11,\n \"max\": 2.6629648345988244e-05,\n \"num_unique_values\": 13959,\n \"samples\": [\n 3.646040269700279e-08\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"C3\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 3.3540844943283726e-11,\n \"min\": 0.0,\n \"max\": 3.1991131965725117e-09,\n \"num_unique_values\": 4,\n \"samples\": [\n 3.1991131965725117e-09\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"D1\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.25368644200727464,\n \"min\": 2.714658148761373e-05,\n \"max\": 0.9851301908493042,\n \"num_unique_values\": 13979,\n \"samples\": [\n 0.0064634778536856174\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"D2\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 5.70214690638657e-08,\n \"min\": 4.071197372279567e-12,\n \"max\": 1.5888683719822438e-06,\n \"num_unique_values\": 13960,\n \"samples\": [\n 7.49811412958934e-09\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"D3\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 3.238652343363632e-11,\n \"min\": 0.0,\n \"max\": 2.805851773146628e-09,\n \"num_unique_values\": 4,\n \"samples\": [\n 2.3405208882110173e-09\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" } }, "metadata": {}, "execution_count": 21 } ] }, { "cell_type": "code", "source": [ "df['A'] = df['A1'] + df['A2'] + df['A3']\n", "df['B'] = df['B1'] + df['B2'] + df['B3']\n", "df['C'] = df['C1'] + df['C2'] + df['C3']\n", "df['D'] = df['D1'] + df['D2'] + df['D3']" ], "metadata": { "id": "z1sa1MNJ2Fo8" }, "id": "z1sa1MNJ2Fo8", "execution_count": 22, "outputs": [] }, { "cell_type": "code", "source": [ "df['ANS'] = df[['A', 'B', 'C', 'D']].idxmax(axis=1)" ], "metadata": { "id": "c1bm0UDqohyP" }, "id": "c1bm0UDqohyP", "execution_count": 23, "outputs": [] }, { "cell_type": "code", "source": [ "df.head(10)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 565 }, "id": "jHJ5DCl4JwXl", "outputId": "a3bbf946-c11d-46e9-838c-45a28a8aa036" }, "id": "jHJ5DCl4JwXl", "execution_count": 24, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ " Input Output \\\n", "1262 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... B \n", "1202 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... A \n", "1289 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... A \n", "1246 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... D \n", "1358 सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब... D \n", "5938 यह प्रश्न निम्नलिखित जानकारी से संबंधित है।\\nस... A \n", "5874 यह प्रश्न निम्नलिखित जानकारी से संबंधित है।\\nस... B \n", "12115 साल 1993 में, एक पशुपालक किसान को 20 एकड़ के स... A \n", "10876 एक भूमि-विकास कंपनी के पास दक्षिणपश्चिम में 40... C \n", "11923 साल 1993 में, एक ज़मींदार के पास एक अंगूर के ब... C \n", "\n", " Sub-Domain Dataset tok A1 A2 A3 \\\n", "1262 college_medicine MMLU 1917 0.616334 3.182060e-07 0.0 \n", "1202 college_medicine MMLU 1894 0.963845 1.845272e-07 0.0 \n", "1289 college_medicine MMLU 1892 0.523709 2.649202e-07 0.0 \n", "1246 college_medicine MMLU 1831 0.205032 8.206729e-08 0.0 \n", "1358 college_medicine MMLU 1830 0.017179 3.866005e-08 0.0 \n", "5938 high_school_world_history MMLU 1313 0.900618 8.246349e-07 0.0 \n", "5874 high_school_world_history MMLU 1292 0.036671 2.693843e-07 0.0 \n", "12115 professional_law MMLU 1279 0.409476 3.885719e-06 0.0 \n", "10876 professional_law MMLU 1277 0.389645 3.015450e-06 0.0 \n", "11923 professional_law MMLU 1266 0.228802 1.258990e-06 0.0 \n", "\n", " B1 B2 ... C2 C3 D1 D2 \\\n", "1262 0.291135 6.803543e-06 ... 1.503099e-07 0.0 0.050592 4.879847e-08 \n", "1202 0.022667 5.339471e-07 ... 5.286792e-08 0.0 0.001642 3.275749e-09 \n", "1289 0.132414 2.675599e-06 ... 1.820768e-07 0.0 0.317646 4.603623e-08 \n", "1246 0.715631 1.632103e-07 ... 2.864430e-07 0.0 0.040373 4.928523e-09 \n", "1358 0.937922 9.774791e-09 ... 3.866005e-08 0.0 0.024995 1.499013e-09 \n", "5938 0.024001 1.359593e-06 ... 5.667628e-07 0.0 0.004726 2.821746e-08 \n", "5874 0.945768 3.458963e-07 ... 4.172305e-07 0.0 0.003010 1.183592e-08 \n", "12115 0.193423 2.079876e-06 ... 2.670613e-06 0.0 0.062795 5.258749e-07 \n", "10876 0.267799 3.015450e-06 ... 1.257027e-06 0.0 0.086942 1.410345e-07 \n", "11923 0.293787 5.248249e-07 ... 4.394305e-06 0.0 0.201917 2.328896e-07 \n", "\n", " D3 A B C D ANS \n", "1262 0.0 0.616334 0.291142 0.027080 0.050592 A \n", "1202 0.0 0.963845 0.022668 0.005731 0.001642 A \n", "1289 0.0 0.523710 0.132417 0.015815 0.317646 A \n", "1246 0.0 0.205032 0.715631 0.024488 0.040373 B \n", "1358 0.0 0.017179 0.937922 0.011807 0.024995 B \n", "5938 0.0 0.900619 0.024002 0.065241 0.004726 A \n", "5874 0.0 0.036672 0.945769 0.008183 0.003010 B \n", "12115 0.0 0.409479 0.193425 0.318903 0.062796 A \n", "10876 0.0 0.389648 0.267802 0.236333 0.086942 A \n", "11923 0.0 0.228803 0.293788 0.259271 0.201917 B \n", "\n", "[10 rows x 22 columns]" ], "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
InputOutputSub-DomainDatasettokA1A2A3B1B2...C2C3D1D2D3ABCDANS
1262सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Bcollege_medicineMMLU19170.6163343.182060e-070.00.2911356.803543e-06...1.503099e-070.00.0505924.879847e-080.00.6163340.2911420.0270800.050592A
1202सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Acollege_medicineMMLU18940.9638451.845272e-070.00.0226675.339471e-07...5.286792e-080.00.0016423.275749e-090.00.9638450.0226680.0057310.001642A
1289सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Acollege_medicineMMLU18920.5237092.649202e-070.00.1324142.675599e-06...1.820768e-070.00.3176464.603623e-080.00.5237100.1324170.0158150.317646A
1246सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Dcollege_medicineMMLU18310.2050328.206729e-080.00.7156311.632103e-07...2.864430e-070.00.0403734.928523e-090.00.2050320.7156310.0244880.040373B
1358सौना का उपयोग करना जिसे \"सौना स्नान\" या सौना ब...Dcollege_medicineMMLU18300.0171793.866005e-080.00.9379229.774791e-09...3.866005e-080.00.0249951.499013e-090.00.0171790.9379220.0118070.024995B
5938यह प्रश्न निम्नलिखित जानकारी से संबंधित है।\\nस...Ahigh_school_world_historyMMLU13130.9006188.246349e-070.00.0240011.359593e-06...5.667628e-070.00.0047262.821746e-080.00.9006190.0240020.0652410.004726A
5874यह प्रश्न निम्नलिखित जानकारी से संबंधित है।\\nस...Bhigh_school_world_historyMMLU12920.0366712.693843e-070.00.9457683.458963e-07...4.172305e-070.00.0030101.183592e-080.00.0366720.9457690.0081830.003010B
12115साल 1993 में, एक पशुपालक किसान को 20 एकड़ के स...Aprofessional_lawMMLU12790.4094763.885719e-060.00.1934232.079876e-06...2.670613e-060.00.0627955.258749e-070.00.4094790.1934250.3189030.062796A
10876एक भूमि-विकास कंपनी के पास दक्षिणपश्चिम में 40...Cprofessional_lawMMLU12770.3896453.015450e-060.00.2677993.015450e-06...1.257027e-060.00.0869421.410345e-070.00.3896480.2678020.2363330.086942A
11923साल 1993 में, एक ज़मींदार के पास एक अंगूर के ब...Cprofessional_lawMMLU12660.2288021.258990e-060.00.2937875.248249e-07...4.394305e-060.00.2019172.328896e-070.00.2288030.2937880.2592710.201917B
\n", "

10 rows × 22 columns

\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", " \n", "\n", "\n", "\n", " \n", "
\n", "\n", "
\n", "
\n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "variable_name": "df" } }, "metadata": {}, "execution_count": 24 } ] }, { "cell_type": "code", "source": [ "accuracy = (df['Output'] == df['ANS']).mean()\n", "print(f\"Accuracy: {accuracy:.4f}\")" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "7o2o5J9Kotv_", "outputId": "339fdd70-9d2a-4b76-f3d2-169f6ccb9495" }, "id": "7o2o5J9Kotv_", "execution_count": 25, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Accuracy: 0.5399\n" ] } ] }, { "cell_type": "code", "source": [ "df['ANS'].value_counts()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 241 }, "id": "BycMopmbIwlE", "outputId": "fce743db-2ed3-4ffa-e5cb-f7eb831e788c" }, "id": "BycMopmbIwlE", "execution_count": 26, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "ANS\n", "B 4453\n", "A 3920\n", "C 2900\n", "D 2769\n", "Name: count, dtype: int64" ], "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
count
ANS
B4453
A3920
C2900
D2769
\n", "

" ] }, "metadata": {}, "execution_count": 26 } ] }, { "cell_type": "code", "source": [ "for sd, group in df.groupby('Sub-Domain'):\n", " accuracy = (group['Output'] == group['ANS']).mean()\n", " print(f\"Sub-domain: {sd}, Accuracy: {accuracy:.4f}\")" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "29c_ZVaRo2O2", "outputId": "97f29e50-2bf7-49a4-9ef9-b7c819577fd0" }, "id": "29c_ZVaRo2O2", "execution_count": 27, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Sub-domain: abstract_algebra, Accuracy: 0.4200\n", "Sub-domain: anatomy, Accuracy: 0.5111\n", "Sub-domain: astronomy, Accuracy: 0.5987\n", "Sub-domain: business_ethics, Accuracy: 0.5100\n", "Sub-domain: clinical_knowledge, Accuracy: 0.5811\n", "Sub-domain: college_biology, Accuracy: 0.6042\n", "Sub-domain: college_chemistry, Accuracy: 0.5100\n", "Sub-domain: college_computer_science, Accuracy: 0.4600\n", "Sub-domain: college_mathematics, Accuracy: 0.3300\n", "Sub-domain: college_medicine, Accuracy: 0.6012\n", "Sub-domain: college_physics, Accuracy: 0.4216\n", "Sub-domain: computer_security, Accuracy: 0.6300\n", "Sub-domain: conceptual_physics, Accuracy: 0.6000\n", "Sub-domain: econometrics, Accuracy: 0.4298\n", "Sub-domain: electrical_engineering, Accuracy: 0.5586\n", "Sub-domain: elementary_mathematics, Accuracy: 0.4841\n", "Sub-domain: formal_logic, Accuracy: 0.4206\n", "Sub-domain: global_facts, Accuracy: 0.4300\n", "Sub-domain: high_school_biology, Accuracy: 0.7258\n", "Sub-domain: high_school_chemistry, Accuracy: 0.5468\n", "Sub-domain: high_school_computer_science, Accuracy: 0.6900\n", "Sub-domain: high_school_european_history, Accuracy: 0.7152\n", "Sub-domain: high_school_geography, Accuracy: 0.6667\n", "Sub-domain: high_school_government_and_politics, Accuracy: 0.6477\n", "Sub-domain: high_school_macroeconomics, Accuracy: 0.5564\n", "Sub-domain: high_school_mathematics, Accuracy: 0.3407\n", "Sub-domain: high_school_microeconomics, Accuracy: 0.5756\n", "Sub-domain: high_school_physics, Accuracy: 0.4305\n", "Sub-domain: high_school_psychology, Accuracy: 0.6826\n", "Sub-domain: high_school_statistics, Accuracy: 0.4861\n", "Sub-domain: high_school_us_history, Accuracy: 0.6716\n", "Sub-domain: high_school_world_history, Accuracy: 0.7553\n", "Sub-domain: human_aging, Accuracy: 0.5336\n", "Sub-domain: human_sexuality, Accuracy: 0.5878\n", "Sub-domain: international_law, Accuracy: 0.6281\n", "Sub-domain: jurisprudence, Accuracy: 0.5926\n", "Sub-domain: logical_fallacies, Accuracy: 0.4049\n", "Sub-domain: machine_learning, Accuracy: 0.4464\n", "Sub-domain: management, Accuracy: 0.6117\n", "Sub-domain: marketing, Accuracy: 0.8077\n", "Sub-domain: medical_genetics, Accuracy: 0.6000\n", "Sub-domain: miscellaneous, Accuracy: 0.6437\n", "Sub-domain: moral_disputes, Accuracy: 0.6012\n", "Sub-domain: moral_scenarios, Accuracy: 0.2458\n", "Sub-domain: nutrition, Accuracy: 0.6405\n", "Sub-domain: philosophy, Accuracy: 0.5981\n", "Sub-domain: prehistory, Accuracy: 0.5710\n", "Sub-domain: professional_accounting, Accuracy: 0.4433\n", "Sub-domain: professional_law, Accuracy: 0.4192\n", "Sub-domain: professional_medicine, Accuracy: 0.5037\n", "Sub-domain: professional_psychology, Accuracy: 0.5212\n", "Sub-domain: public_relations, Accuracy: 0.6636\n", "Sub-domain: security_studies, Accuracy: 0.6694\n", "Sub-domain: sociology, Accuracy: 0.6915\n", "Sub-domain: us_foreign_policy, Accuracy: 0.5900\n", "Sub-domain: virology, Accuracy: 0.4578\n", "Sub-domain: world_religions, Accuracy: 0.6725\n" ] } ] }, { "cell_type": "code", "source": [ "print(\"CLICK CTRl+S and wait for 2 sec\")\n", "name = name.split('/')[-1]\n", "name = name + \"-MMLU-H.ipynb\"\n", "print(\"NOTEBOOK NAME SHOULD BE : \", name)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "m7vhP5Obk-sw", "outputId": "215d44b7-e03e-4908-a939-155b5bd9acf4" }, "id": "m7vhP5Obk-sw", "execution_count": 28, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "CLICK CTRl+S and wait for 2 sec\n", "NOTEBOOK NAME SHOULD BE : HINDI-GEMMA-9B-B20-MMLU-H.ipynb\n" ] } ] }, { "cell_type": "code", "source": [], "metadata": { "id": "c_5zVBU12crT" }, "id": "c_5zVBU12crT", "execution_count": 28, "outputs": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" }, "colab": { "provenance": [], "machine_shape": "hm", "gpuType": "A100" }, "accelerator": "GPU", "widgets": { "application/vnd.jupyter.widget-state+json": { "ffbb5c6a3b7f41a2a20769204dde9281": { "model_module": "@jupyter-widgets/controls", "model_name": "VBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "VBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "VBoxView", "box_style": "", "children": [ "IPY_MODEL_1f309cae286e4bbebc79393e43b32205", "IPY_MODEL_a66cb4bca4fb45c893549fc9bace8150", "IPY_MODEL_9f626afdd1ef42adb413b98088ef49da", "IPY_MODEL_9135fa47675a417eac79514c9c071c1b", "IPY_MODEL_73a82d5ee0b14f4599904fda0aff3d67" ], "layout": "IPY_MODEL_5b9b62cc01444ce98148697c41a05c90" } }, "1f309cae286e4bbebc79393e43b32205": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_2518a6ae25b7427faa155115efa4d921", "placeholder": "​", "style": "IPY_MODEL_584e7cacfe634c23b9902c4e655219bd", "value": "

Copy a token from your Hugging Face\ntokens page and paste it below.
Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file.
" } }, "a66cb4bca4fb45c893549fc9bace8150": { "model_module": "@jupyter-widgets/controls", "model_name": "PasswordModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "PasswordModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "PasswordView", "continuous_update": true, "description": "Token:", "description_tooltip": null, "disabled": false, "layout": "IPY_MODEL_d672ac2ea2814aa18b41501e6a0aee25", "placeholder": "​", "style": "IPY_MODEL_d902c2ff97e240b4ae6783d8824e3822", "value": "" } }, "9f626afdd1ef42adb413b98088ef49da": { "model_module": "@jupyter-widgets/controls", "model_name": "CheckboxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "CheckboxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "CheckboxView", "description": "Add token as git credential?", "description_tooltip": null, "disabled": false, "indent": true, "layout": "IPY_MODEL_966c62fdbf0843f498e2aa226e9e761c", "style": "IPY_MODEL_7e50b448b5544a8cb6a38250623c0331", "value": true } }, "9135fa47675a417eac79514c9c071c1b": { "model_module": "@jupyter-widgets/controls", "model_name": "ButtonModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ButtonModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ButtonView", "button_style": "", "description": "Login", "disabled": false, "icon": "", "layout": "IPY_MODEL_30b61336ce554e4284bfc64f7fbbbae8", "style": "IPY_MODEL_1385025409a74dfda9c3d2c9c75b914a", "tooltip": "" } }, "73a82d5ee0b14f4599904fda0aff3d67": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_382ca19ee53a42b2918c6cf9188fa1ca", "placeholder": "​", "style": "IPY_MODEL_42a33944551b46769ac206e4f15bc77e", "value": "\nPro Tip: If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks.
" } }, "5b9b62cc01444ce98148697c41a05c90": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": "center", "align_self": null, "border": null, "bottom": null, "display": "flex", "flex": null, "flex_flow": "column", "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": "50%" } }, "2518a6ae25b7427faa155115efa4d921": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "584e7cacfe634c23b9902c4e655219bd": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "d672ac2ea2814aa18b41501e6a0aee25": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d902c2ff97e240b4ae6783d8824e3822": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "966c62fdbf0843f498e2aa226e9e761c": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "7e50b448b5544a8cb6a38250623c0331": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "30b61336ce554e4284bfc64f7fbbbae8": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "1385025409a74dfda9c3d2c9c75b914a": { "model_module": "@jupyter-widgets/controls", "model_name": "ButtonStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ButtonStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "button_color": null, "font_weight": "" } }, "382ca19ee53a42b2918c6cf9188fa1ca": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "42a33944551b46769ac206e4f15bc77e": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "f740f415ad4844ddac0c986178beb78b": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_b65ff97b1b5e432f9ed61a2678f2d638", "IPY_MODEL_4c904bd27f5441aba7f2bcf3ca785514", "IPY_MODEL_9146a0a9dc374f579a081605b5f6f6df" ], "layout": "IPY_MODEL_450d892e1b8449ad8ddf2d66cb30047a" } }, "b65ff97b1b5e432f9ed61a2678f2d638": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_7a95604acad14604bb86ab3470f78aee", "placeholder": "​", "style": "IPY_MODEL_ae73ab6cfd404d468f11300b14440e42", "value": "Loading checkpoint shards: 100%" } }, "4c904bd27f5441aba7f2bcf3ca785514": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_d5267962fb9a4fc8a190625e2feda62e", "max": 4, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_fbb06183005448e7b7ddb57898c105fd", "value": 4 } }, "9146a0a9dc374f579a081605b5f6f6df": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_7d1214a6856d46a2a9e13f85a9a25e11", "placeholder": "​", "style": "IPY_MODEL_5323089c028c411185dbd93da6e5c702", "value": " 4/4 [00:08<00:00,  1.94s/it]" } }, "450d892e1b8449ad8ddf2d66cb30047a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "7a95604acad14604bb86ab3470f78aee": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ae73ab6cfd404d468f11300b14440e42": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "d5267962fb9a4fc8a190625e2feda62e": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "fbb06183005448e7b7ddb57898c105fd": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "7d1214a6856d46a2a9e13f85a9a25e11": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5323089c028c411185dbd93da6e5c702": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "nbformat": 4, "nbformat_minor": 5 }