diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,37488 +1,88 @@ -2024-04-25 13:45:18,234 INFO StreamThr :156911 [internal.py:wandb_internal():86] W&B internal server running at pid: 156911, started at: 2024-04-25 13:45:18.233208 -2024-04-25 13:45:18,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status -2024-04-25 13:45:18,241 INFO WriterThread:156911 [datastore.py:open_for_write():87] open: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/run-etajcxpg.wandb -2024-04-25 13:45:18,243 DEBUG SenderThread:156911 [sender.py:send():379] send: header -2024-04-25 13:45:18,269 DEBUG SenderThread:156911 [sender.py:send():379] send: run -2024-04-25 13:45:18,501 INFO SenderThread:156911 [dir_watcher.py:__init__():211] watching files in: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files -2024-04-25 13:45:18,501 INFO SenderThread:156911 [sender.py:_start_run_threads():1124] run started: etajcxpg with start time 1714052718.239809 -2024-04-25 13:45:18,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: check_version -2024-04-25 13:45:18,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: check_version -2024-04-25 13:45:18,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: run_start -2024-04-25 13:45:18,625 DEBUG HandlerThread:156911 [system_info.py:__init__():26] System info init -2024-04-25 13:45:18,625 DEBUG HandlerThread:156911 [system_info.py:__init__():41] System info init done -2024-04-25 13:45:18,625 INFO HandlerThread:156911 [system_monitor.py:start():194] Starting system monitor -2024-04-25 13:45:18,625 INFO SystemMonitor:156911 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-04-25 13:45:18,625 INFO HandlerThread:156911 [system_monitor.py:probe():214] Collecting system info -2024-04-25 13:45:18,625 INFO SystemMonitor:156911 [interfaces.py:start():190] Started cpu monitoring -2024-04-25 13:45:18,626 INFO SystemMonitor:156911 [interfaces.py:start():190] Started disk monitoring -2024-04-25 13:45:18,627 INFO SystemMonitor:156911 [interfaces.py:start():190] Started gpu monitoring -2024-04-25 13:45:18,627 INFO SystemMonitor:156911 [interfaces.py:start():190] Started memory monitoring -2024-04-25 13:45:18,627 INFO SystemMonitor:156911 [interfaces.py:start():190] Started network monitoring -2024-04-25 13:45:18,674 DEBUG HandlerThread:156911 [system_info.py:probe():150] Probing system -2024-04-25 13:45:18,677 DEBUG HandlerThread:156911 [system_info.py:_probe_git():135] Probing git -2024-04-25 13:45:18,698 DEBUG HandlerThread:156911 [system_info.py:_probe_git():143] Probing git done -2024-04-25 13:45:18,698 DEBUG HandlerThread:156911 [system_info.py:probe():198] Probing system done -2024-04-25 13:45:18,698 DEBUG HandlerThread:156911 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1048-aws-x86_64-with-glibc2.31', 'python': '3.11.9', 'heartbeatAt': '2024-04-25T13:45:18.674598', 'startedAt': '2024-04-25T13:45:18.212334', 'docker': None, 'cuda': None, 'args': ('./config_full.yaml',), 'state': 'running', 'program': '/fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/./run_sft.py', 'codePathLocal': 'run_sft.py', 'codePath': 'run_sft.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/distil-zephyr-1.5b-ssft-ultrachat', 'commit': 'cbea69c6b95c970317a1e47c3f614b55b33f8ed9'}, 'email': None, 'root': '/fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat', 'host': 'ip-26-0-167-177', 'username': 'sanchit', 'executable': '/fsx/sanchit/miniconda3/envs/alignment/bin/python', 'cpu_count': 96, 'cpu_count_logical': 96, 'cpu_freq': {'current': 2725.4579687499986, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3597.164, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3597.955, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3596.13, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3597.91, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3592.057, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3267.919, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3597.091, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3597.915, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 290.7472343444824, 'used': 58.59657669067383}}, 'gpu': 'NVIDIA H100 80GB HBM3', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}], 'memory': {'total': 1999.9855155944824}} -2024-04-25 13:45:18,698 INFO HandlerThread:156911 [system_monitor.py:probe():224] Finished collecting system info -2024-04-25 13:45:18,698 INFO HandlerThread:156911 [system_monitor.py:probe():227] Publishing system info -2024-04-25 13:45:18,698 DEBUG HandlerThread:156911 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment -2024-04-25 13:45:19,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/conda-environment.yaml -2024-04-25 13:45:22,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/conda-environment.yaml -2024-04-25 13:45:22,523 DEBUG HandlerThread:156911 [system_info.py:_save_conda():222] Saving conda packages done -2024-04-25 13:45:22,526 INFO HandlerThread:156911 [system_monitor.py:probe():229] Finished publishing system info -2024-04-25 13:45:22,561 DEBUG SenderThread:156911 [sender.py:send():379] send: files -2024-04-25 13:45:22,561 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now -2024-04-25 13:45:22,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: python_packages -2024-04-25 13:45:22,707 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: python_packages -2024-04-25 13:45:22,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:45:22,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:45:22,710 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:45:22,829 DEBUG SenderThread:156911 [sender.py:send():379] send: telemetry -2024-04-25 13:45:22,830 DEBUG SenderThread:156911 [sender.py:send():379] send: config -2024-04-25 13:45:22,831 DEBUG SenderThread:156911 [sender.py:send():379] send: metric -2024-04-25 13:45:22,831 DEBUG SenderThread:156911 [sender.py:send():379] send: telemetry -2024-04-25 13:45:22,831 DEBUG SenderThread:156911 [sender.py:send():379] send: metric -2024-04-25 13:45:22,831 WARNING SenderThread:156911 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) -2024-04-25 13:45:22,831 DEBUG SenderThread:156911 [sender.py:send():379] send: telemetry -2024-04-25 13:45:22,840 INFO wandb-upload_0:156911 [upload_job.py:push():131] Uploaded file /tmp/tmpcjtpmhukwandb/bsh2e287-wandb-metadata.json -2024-04-25 13:45:23,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:23,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/requirements.txt -2024-04-25 13:45:23,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-metadata.json -2024-04-25 13:45:23,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:45:25,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:26,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:45:26,068 DEBUG SenderThread:156911 [sender.py:send():379] send: metric -2024-04-25 13:45:26,069 DEBUG SenderThread:156911 [sender.py:send():379] send: metric -2024-04-25 13:45:26,069 DEBUG SenderThread:156911 [sender.py:send():379] send: metric -2024-04-25 13:45:26,069 DEBUG SenderThread:156911 [sender.py:send():379] send: metric -2024-04-25 13:45:26,069 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:45:26,070 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:45:26,071 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:45:26,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:45:27,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:28,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:29,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:45:31,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:33,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:34,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:45:35,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:37,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:37,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:45:37,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:45:37,708 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:45:39,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:45:39,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:41,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:43,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:45,235 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:45:45,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:47,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:49,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:51,047 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:45:51,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:52,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/config.yaml -2024-04-25 13:45:52,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:45:52,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:45:52,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:45:53,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:55,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:57,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:45:57,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:45:59,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:01,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:03,530 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:46:03,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:05,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:07,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:07,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:46:07,708 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:46:07,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:46:08,838 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:46:09,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:11,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:13,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:46:13,191 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:46:13,191 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:46:13,193 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:46:13,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:13,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:46:14,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:46:14,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:17,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:18,628 DEBUG SystemMonitor:156911 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-04-25 13:46:18,631 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:46:19,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:19,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:46:21,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:22,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:46:22,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:46:22,708 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:46:23,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:25,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:25,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:46:27,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:29,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:31,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:46:31,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:33,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:35,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:37,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:46:37,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:37,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:46:37,708 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:46:37,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:46:39,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:41,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:42,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:46:43,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:45,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:47,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:48,633 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:46:48,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:46:49,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:51,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:52,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:46:52,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:46:52,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:46:53,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:54,429 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:46:55,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:57,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:46:59,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:00,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:47:01,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:47:01,103 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:47:01,103 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:47:01,105 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:47:01,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:01,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:47:02,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:05,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:05,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:47:07,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:07,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:47:07,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:47:07,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:47:09,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:11,598 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:47:11,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:13,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:15,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:17,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:47:17,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:18,635 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:47:19,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:21,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:22,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:47:22,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:47:22,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:47:22,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:47:23,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:25,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:27,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:28,740 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:47:29,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:31,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:33,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:34,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:47:35,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:37,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:37,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:47:37,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:47:37,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:47:39,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:40,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:47:41,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:43,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:45,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:45,738 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:47:47,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:48,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:47:48,550 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:47:48,550 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:47:48,551 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:47:48,636 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:47:48,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:47:49,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:50,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:51,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:47:52,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:47:52,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:47:52,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:47:53,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:55,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:57,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:47:57,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:47:59,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:01,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:02,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:48:03,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:05,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:07,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:07,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:48:07,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:48:07,710 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:48:07,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:48:09,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:11,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:13,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:48:13,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:15,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:17,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:18,639 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:48:18,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:48:19,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:21,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:22,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:48:22,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:48:22,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:48:23,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:23,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:48:25,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:27,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:29,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:48:29,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:31,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:33,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:34,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:48:35,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:48:35,705 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:48:35,705 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:48:35,707 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:48:35,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:35,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:48:37,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:48:37,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:48:37,710 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:48:37,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:38,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:40,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:48:41,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:43,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:45,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:46,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:48:47,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:48,640 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:48:49,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:51,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:51,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:48:52,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:48:52,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:48:52,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:48:53,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:55,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:57,363 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:48:57,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:48:59,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:01,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:03,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:49:03,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:05,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:07,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:07,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:49:07,793 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:49:07,794 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:49:08,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:49:09,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:11,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:13,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:14,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:49:15,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:17,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:18,642 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:49:19,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:19,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:49:21,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:22,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:49:22,563 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:49:22,563 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:49:22,565 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:49:22,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:49:22,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:49:22,773 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:49:22,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:49:23,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:24,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:25,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:49:27,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:29,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:30,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:49:31,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:33,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:35,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:36,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:49:37,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:37,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:49:37,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:49:37,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:49:39,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:41,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:42,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:49:43,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:45,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:47,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:49:47,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:48,644 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:49:49,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:51,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:52,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:49:52,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:49:52,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:49:52,931 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:49:53,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:55,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:57,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:49:58,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:49:59,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:01,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:03,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:04,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:50:05,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:07,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:50:07,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:50:07,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:50:07,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:09,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:50:09,103 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:50:09,103 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:50:09,105 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:50:09,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:09,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:50:10,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:50:10,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:13,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:15,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:50:15,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:17,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:18,646 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:50:19,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:21,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:50:21,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:22,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:50:22,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:50:22,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:50:23,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:25,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:26,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:50:27,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:29,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:31,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:32,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:50:33,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:35,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:37,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:50:37,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:50:37,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:50:37,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:50:37,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:39,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:41,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:43,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:50:43,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:45,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:47,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:48,648 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:50:48,649 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:50:49,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:51,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:52,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:50:52,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:50:52,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:50:53,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:54,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:50:55,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:50:55,391 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:50:55,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:50:55,393 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:50:55,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:55,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:50:56,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:50:59,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:00,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:51:01,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:03,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:05,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:51:05,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:07,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:51:07,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:51:07,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:51:07,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:09,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:11,186 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:51:11,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:13,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:15,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:16,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:51:17,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:18,650 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:51:19,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:21,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:22,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:51:22,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:51:22,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:51:22,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:51:23,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:25,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:27,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:51:27,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:29,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:31,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:33,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:51:33,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:35,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:37,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:51:37,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:51:37,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:51:37,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:38,931 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:51:39,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:41,553 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:51:41,554 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:51:41,555 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:51:41,556 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:51:41,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:41,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:51:42,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:44,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:51:45,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:47,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:48,663 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:51:49,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:51:49,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:51,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:52,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:51:52,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:51:52,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:51:53,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:55,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:51:55,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:57,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:51:59,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:00,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:52:01,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:03,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:05,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:06,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:52:07,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:52:07,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:52:07,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:52:07,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:09,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:11,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:52:11,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:13,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:15,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:17,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:52:17,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:18,654 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:52:19,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:21,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:22,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:52:22,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:52:22,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:52:22,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:52:23,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:25,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:27,553 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:52:27,554 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:52:27,555 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:52:27,556 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:52:27,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:27,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:52:28,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:52:28,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:31,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:33,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:34,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:52:35,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:37,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:52:37,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:52:37,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:52:37,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:39,536 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:52:39,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:41,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:43,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:45,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:52:45,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:47,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:48,656 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:52:49,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:50,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:52:51,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:52,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:52:52,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:52:52,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:52:53,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:55,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:56,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:52:57,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:52:59,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:01,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:53:01,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:03,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:04,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:06,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:07,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:53:07,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:53:07,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:53:07,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:53:08,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:10,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:12,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:53:12,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:13,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:53:13,350 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:53:13,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:53:13,352 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:53:13,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:53:14,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:16,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:17,977 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:53:18,658 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:53:18,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:20,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:22,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:53:22,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:53:22,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:53:22,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:23,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:53:24,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:26,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:28,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:53:29,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:31,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:33,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:34,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:53:35,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:37,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:37,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:53:37,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:53:37,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:53:39,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:39,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:53:41,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:43,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:45,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:45,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:53:47,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:48,715 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:53:49,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:51,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:51,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:53:52,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:53:52,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:53:52,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:53:53,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:55,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:56,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:53:57,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:59,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:53:59,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:53:59,320 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:53:59,320 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:53:59,322 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:54:00,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:54:01,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:02,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:54:03,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:05,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:07,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:07,590 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:54:07,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:54:07,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:54:07,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:54:09,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:11,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:13,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:13,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:54:15,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:17,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:18,567 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:54:18,662 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:54:19,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:21,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:22,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:54:22,776 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:54:22,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:54:23,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:24,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:54:25,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:27,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:29,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:29,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:54:31,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:33,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:35,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:54:35,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:37,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:37,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:54:37,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:54:37,847 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:54:39,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:40,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:54:41,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:43,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:45,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:45,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:54:45,108 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:54:45,108 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:54:45,110 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:54:46,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:54:46,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:54:47,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:48,664 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:54:49,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:51,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:51,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:54:52,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:54:52,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:54:52,826 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:54:53,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:55,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:57,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:54:57,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:54:59,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:01,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:02,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:55:03,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:05,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:07,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:07,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:55:07,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:55:07,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:55:07,826 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:55:09,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:11,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:13,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:13,499 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:55:15,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:17,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:18,667 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:55:18,669 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:55:19,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:21,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:22,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:55:22,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:55:22,826 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:55:23,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:24,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:55:25,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:27,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:29,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:30,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:55:30,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:55:30,954 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:55:30,954 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:55:30,957 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:55:31,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:31,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:55:32,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:33,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:35,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:35,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:55:37,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:37,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:55:37,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:55:37,826 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:55:39,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:41,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:55:41,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:43,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:45,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:46,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:55:47,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:48,684 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:55:49,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:51,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:51,941 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:55:52,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:55:52,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:55:52,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:55:53,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:55,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:57,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:55:57,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:55:59,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:01,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:02,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:56:03,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:05,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:07,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:07,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:56:07,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:56:07,866 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:56:07,976 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:56:09,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:11,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:13,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:13,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:56:15,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:16,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:56:16,445 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:56:16,446 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:56:16,448 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:56:17,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:17,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:56:18,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:18,672 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:56:19,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:19,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:56:21,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:22,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:56:22,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:56:22,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:56:23,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:24,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:56:25,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:27,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:29,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:30,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:56:31,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:33,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:35,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:35,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:56:37,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:37,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:56:37,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:56:37,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:56:39,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:41,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:56:41,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:43,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:45,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:46,597 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:56:47,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:48,674 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:56:49,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:51,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:52,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:56:52,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:56:52,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:56:52,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:56:53,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:55,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:57,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:56:57,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:56:59,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:01,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:01,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:57:01,995 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:57:01,996 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:57:01,998 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:57:02,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:57:02,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:57:03,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:04,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:05,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:07,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:07,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:57:07,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:57:07,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:57:08,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:57:09,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:11,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:13,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:14,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:57:15,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:17,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:18,676 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:57:19,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:20,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:57:21,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:22,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:57:22,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:57:22,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:57:23,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:25,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:25,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:57:27,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:29,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:31,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:57:31,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:33,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:35,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:36,553 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:57:37,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:37,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:57:37,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:57:37,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:57:39,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:41,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:42,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:57:43,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:45,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:47,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:57:47,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:57:47,507 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:57:47,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:57:47,508 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:57:47,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:47,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:57:48,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:48,677 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:57:49,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:51,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:52,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:57:52,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:57:52,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:57:52,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:57:53,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:55,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:57,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:57:58,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:57:59,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:01,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:03,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:03,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:58:05,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:07,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:07,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:58:07,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:58:07,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:58:09,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:58:09,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:11,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:13,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:14,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:58:15,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:17,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:18,680 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:58:19,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:20,231 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:58:21,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:22,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:58:22,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:58:22,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:58:23,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:25,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:25,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:58:27,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:29,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:31,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:58:31,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:32,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:58:32,984 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:58:32,984 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:58:32,986 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:58:33,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:33,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:58:34,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:35,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:36,606 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:58:37,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:37,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:58:37,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:58:37,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:58:39,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:41,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:42,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:58:43,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:45,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:47,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:58:47,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:48,682 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:58:49,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:51,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:52,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:58:52,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:58:52,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:58:52,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:58:53,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:55,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:57,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:58:58,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:58:59,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:01,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:03,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:03,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:59:05,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:07,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:07,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:59:07,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:59:07,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:59:08,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:59:09,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:11,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:13,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:14,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:59:15,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:17,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:18,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 13:59:18,498 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 13:59:18,498 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 13:59:18,499 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 13:59:18,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 13:59:18,683 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:59:19,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:19,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:59:20,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:21,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:22,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:59:22,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:59:22,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:59:23,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:24,919 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:59:25,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:27,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:29,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:30,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:59:31,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:33,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:35,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:35,812 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:59:37,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:37,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:59:37,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:59:37,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:59:39,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:41,256 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:59:41,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:43,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:45,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:46,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:59:47,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:48,686 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 13:59:49,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:51,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:52,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:59:52,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 13:59:52,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 13:59:52,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 13:59:53,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:55,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:57,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 13:59:57,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 13:59:59,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:01,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:03,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:00:03,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:03,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:00:03,924 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:00:03,924 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:00:03,925 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:00:04,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:00:05,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:06,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:07,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:07,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:00:07,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:00:07,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:00:09,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:00:09,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:11,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:13,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:14,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:00:15,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:17,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:18,698 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:00:19,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:20,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:00:21,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:22,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:00:22,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:00:22,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:00:23,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:25,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:00:25,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:27,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:29,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:31,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:00:31,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:33,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:35,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:36,469 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:00:37,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:37,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:00:37,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:00:37,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:00:39,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:41,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:42,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:00:43,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:45,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:47,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:00:47,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:48,690 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:00:49,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:00:49,404 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:00:49,405 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:00:49,406 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:00:49,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:49,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:00:50,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:51,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:52,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:00:52,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:00:52,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:00:52,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:00:53,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:55,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:57,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:00:58,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:00:59,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:01,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:03,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:04,015 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:01:05,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:07,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:07,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:01:07,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:01:07,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:01:09,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:01:09,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:11,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:13,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:14,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:01:15,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:17,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:18,692 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:01:19,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:01:19,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:21,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:22,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:01:22,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:01:22,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:01:23,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:25,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:01:25,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:27,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:29,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:30,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:01:31,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:33,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:35,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:01:35,076 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:01:35,076 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:01:35,078 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:01:35,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:35,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:01:36,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:01:36,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:37,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:37,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:01:37,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:01:37,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:01:39,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:41,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:01:41,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:43,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:45,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:47,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:01:47,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:48,694 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:01:49,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:51,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:52,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:01:52,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:01:52,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:01:52,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:01:53,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:55,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:57,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:01:57,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:01:59,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:01,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:03,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:02:03,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:05,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:07,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:07,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:02:07,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:02:07,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:02:09,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:02:09,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:11,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:13,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:14,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:02:15,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:17,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:18,793 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:02:19,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:19,794 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:02:20,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:02:20,646 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:02:20,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:02:20,648 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:02:20,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:02:21,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:22,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:22,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:02:22,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:02:22,924 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:02:23,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:25,263 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:02:25,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:27,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:29,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:30,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:02:31,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:33,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:35,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:36,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:02:37,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:37,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:02:37,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:02:37,924 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:02:39,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:41,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:02:41,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:43,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:45,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:47,117 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:02:47,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:48,698 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:02:49,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:51,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:52,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:02:52,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:02:52,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:02:52,924 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:02:53,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:55,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:57,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:02:58,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:02:59,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:01,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:03,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:03:03,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:05,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:06,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:03:06,206 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:03:06,206 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:03:06,208 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:03:06,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:03:07,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:07,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:03:07,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:03:07,924 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:03:08,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:09,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:03:09,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:11,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:13,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:14,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:03:15,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:17,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:18,700 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:03:19,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:19,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:03:21,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:22,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:03:22,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:03:22,924 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:03:23,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:25,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:03:25,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:27,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:29,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:30,766 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:03:31,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:33,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:35,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:36,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:03:37,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:37,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:03:37,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:03:37,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:03:39,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:41,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:03:41,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:43,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:45,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:47,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:03:47,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:48,702 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:03:49,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:51,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:03:51,683 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:03:51,683 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:03:51,684 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:03:51,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:51,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:03:52,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:03:52,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:03:52,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:52,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:03:52,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:03:53,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:55,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:57,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:03:58,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:03:59,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:01,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:03,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:04:03,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:05,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:07,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:04:07,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:07,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:04:07,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:04:09,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:04:09,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:11,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:13,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:14,594 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:04:15,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:17,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:18,805 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:04:19,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:20,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:04:21,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:22,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:04:22,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:04:22,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:04:23,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:25,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:04:25,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:27,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:29,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:31,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:04:31,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:33,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:35,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:36,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:04:37,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:04:37,391 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:04:37,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:04:37,393 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:04:37,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:04:37,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:04:37,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:04:37,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:37,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:04:38,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:39,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:41,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:42,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:04:43,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:45,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:47,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:04:47,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:48,707 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:04:49,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:51,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:52,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:04:52,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:04:52,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:04:52,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:04:53,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:55,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:57,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:04:58,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:04:59,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:01,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:03,889 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:05:03,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:05,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:07,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:05:07,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:05:07,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:05:07,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:09,356 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:05:09,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:11,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:13,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:14,831 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:05:16,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:18,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:18,709 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:05:20,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:20,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:05:22,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:22,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:05:22,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:05:22,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:05:23,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:05:23,056 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:05:23,056 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:05:23,058 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:05:23,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:05:24,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:25,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:25,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:05:26,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:28,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:30,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:31,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:05:32,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:34,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:35,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:37,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:05:37,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:37,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:05:37,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:05:37,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:05:39,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:41,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:42,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:05:43,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:45,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:47,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:47,949 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:05:48,712 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:05:49,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:51,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:52,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:05:52,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:05:52,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:05:53,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:05:53,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:55,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:57,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:05:58,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:05:59,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:01,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:03,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:04,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:06:05,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:07,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:07,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:06:07,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:06:07,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:06:08,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:06:08,814 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:06:08,815 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:06:08,817 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:06:09,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:09,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:06:09,818 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:06:11,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:13,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:15,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:15,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:06:17,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:18,727 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:06:19,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:20,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:06:21,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:22,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:06:22,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:06:22,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:06:23,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:25,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:26,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:06:27,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:29,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:31,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:31,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:06:33,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:35,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:37,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:37,256 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:06:37,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:06:37,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:06:37,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:06:39,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:41,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:42,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:06:43,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:45,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:47,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:48,165 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:06:48,716 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:06:49,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:51,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:52,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:06:52,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:06:52,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:06:53,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:54,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:06:54,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:06:54,491 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:06:54,491 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:06:54,493 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:06:55,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:55,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:06:56,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:57,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:06:59,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:06:59,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:01,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:03,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:04,606 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:07:05,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:07,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:07,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:07:07,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:07:07,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:07:09,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:10,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:07:11,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:13,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:15,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:15,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:07:17,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:18,718 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:07:19,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:20,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:07:21,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:22,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:07:22,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:07:22,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:07:23,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:25,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:26,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:07:27,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:29,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:31,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:31,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:07:33,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:35,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:37,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:37,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:07:37,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:07:37,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:07:37,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:07:39,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:39,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:07:39,966 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:07:39,967 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:07:39,968 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:07:40,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:07:41,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:42,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:42,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:07:43,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:45,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:47,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:48,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:07:48,720 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:07:49,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:51,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:52,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:07:52,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:07:52,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:07:53,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:54,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:07:55,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:57,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:07:59,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:07:59,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:01,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:03,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:04,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:08:05,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:07,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:07,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:08:07,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:08:07,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:08:09,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:10,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:08:11,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:13,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:15,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:15,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:08:17,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:18,722 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:08:19,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:20,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:08:21,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:22,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:08:22,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:08:22,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:08:23,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:25,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:25,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:08:25,442 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:08:25,443 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:08:25,445 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:08:26,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:26,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:08:26,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:08:27,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:29,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:31,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:31,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:08:33,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:35,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:37,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:08:37,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:37,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:08:37,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:08:37,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:08:39,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:41,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:42,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:08:43,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:45,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:47,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:48,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:08:48,758 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:08:49,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:51,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:52,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:08:52,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:08:52,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:08:53,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:54,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:08:55,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:57,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:59,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:08:59,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:09:01,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:03,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:05,231 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:09:05,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:07,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:07,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:09:07,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:09:07,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:09:09,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:10,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:09:10,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:09:10,695 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:09:10,695 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:09:10,697 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:09:11,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:11,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:09:12,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:13,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:15,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:16,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:09:17,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:18,760 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:09:19,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:21,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:21,567 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:09:22,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:09:22,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:09:22,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:09:23,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:25,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:27,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:09:27,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:29,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:31,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:32,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:09:33,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:35,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:37,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:37,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:09:37,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:09:37,977 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:09:37,977 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:09:39,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:41,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:43,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:09:43,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:45,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:47,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:48,762 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:09:48,762 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:09:49,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:51,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:52,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:09:52,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:09:52,947 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:09:53,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:54,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:09:55,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:56,046 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:09:56,048 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:09:56,048 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:09:56,050 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:09:56,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:09:57,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:58,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:59,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:09:59,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:10:01,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:03,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:05,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:10:05,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:07,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:07,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:10:07,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:10:07,947 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:10:09,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:10,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:10:11,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:13,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:15,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:15,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:10:17,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:18,764 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:10:19,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:21,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:10:21,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:22,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:10:22,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:10:22,947 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:10:23,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:25,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:26,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:10:27,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:29,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:31,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:32,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:10:33,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:35,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:37,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:37,592 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:10:37,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:10:37,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:10:37,947 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:10:39,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:41,233 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:10:41,235 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:10:41,235 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:10:41,237 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:10:41,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:41,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:10:42,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:43,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:10:43,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:45,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:47,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:48,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:10:48,778 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:10:49,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:51,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:52,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:10:52,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:10:52,947 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:10:53,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:53,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:10:55,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:57,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:10:59,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:10:59,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:01,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:03,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:04,782 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:11:05,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:07,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:07,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:11:07,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:11:07,947 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:11:09,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:10,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:11:11,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:13,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:15,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:15,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:11:17,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:18,768 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:11:19,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:21,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:11:21,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:22,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:11:22,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:11:22,965 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:11:23,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:25,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:26,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:11:26,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:11:26,514 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:11:26,515 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:11:26,516 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:11:26,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:11:27,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:28,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:29,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:31,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:31,896 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:11:33,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:35,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:37,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:11:37,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:37,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:11:37,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:11:37,958 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:11:39,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:41,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:42,736 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:11:43,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:45,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:47,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:48,165 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:11:48,770 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:11:49,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:51,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:52,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:11:52,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:11:52,958 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:11:53,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:53,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:11:55,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:57,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:11:59,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:11:59,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:01,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:03,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:04,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:12:05,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:07,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:07,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:12:07,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:12:07,958 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:12:09,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:09,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:12:11,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:11,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:12:11,691 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:12:11,691 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:12:11,693 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:12:12,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:12,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:12:13,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:15,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:12:15,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:17,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:18,773 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:12:19,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:20,675 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:12:21,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:22,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:12:22,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:12:22,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:12:23,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:25,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:26,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:12:27,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:29,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:31,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:12:31,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:33,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:35,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:36,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:12:37,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:37,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:12:37,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:12:37,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:12:39,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:41,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:42,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:12:43,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:45,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:47,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:47,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:12:48,787 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:12:49,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:51,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:52,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:12:52,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:12:52,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:12:53,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:12:53,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:55,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:57,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:12:57,063 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:12:57,064 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:12:57,065 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:12:57,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:57,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:12:58,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:12:58,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:12:59,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:01,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:03,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:04,243 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:13:05,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:07,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:07,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:13:07,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:13:07,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:13:09,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:09,655 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:13:11,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:13,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:15,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:13:15,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:17,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:18,776 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:13:19,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:20,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:13:21,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:22,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:13:22,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:13:22,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:13:23,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:25,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:25,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:13:27,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:29,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:31,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:13:31,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:33,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:35,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:36,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:13:37,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:37,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:13:37,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:13:37,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:13:39,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:41,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:42,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:13:42,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:13:42,253 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:13:42,254 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:13:42,255 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:13:42,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:13:43,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:44,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:45,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:47,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:13:47,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:48,779 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:13:49,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:51,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:52,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:13:52,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:13:52,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:13:52,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:13:53,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:55,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:57,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:13:58,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:13:59,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:01,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:03,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:03,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:14:05,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:07,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:07,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:14:07,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:14:07,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:14:09,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:14:09,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:11,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:13,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:14,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:14:15,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:17,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:18,781 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:14:19,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:19,782 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:14:21,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:22,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:14:22,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:14:22,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:14:23,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:25,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:14:25,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:27,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:14:27,380 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:14:27,380 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:14:27,382 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:14:27,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:27,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:14:28,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:29,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:30,961 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:14:31,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:33,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:35,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:36,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:14:37,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:37,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:14:37,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:14:37,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:14:39,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:41,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:41,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:14:43,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:45,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:47,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:14:47,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:48,807 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:14:49,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:51,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:52,573 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:14:52,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:14:52,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:14:52,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:14:53,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:55,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:57,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:14:57,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:14:59,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:01,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:03,363 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:15:03,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:05,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:07,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:07,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:15:07,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:15:07,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:15:08,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:15:09,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:11,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:12,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:15:12,390 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:15:12,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:15:12,393 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:15:12,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:15:13,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:14,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:15:14,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:15,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:17,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:18,785 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:15:19,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:15:19,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:21,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:22,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:15:22,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:15:22,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:15:23,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:25,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:15:25,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:27,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:29,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:30,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:15:31,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:33,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:35,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:15:35,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:37,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:15:37,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:37,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:15:37,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:15:39,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:41,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:15:41,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:43,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:45,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:46,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:15:47,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:48,787 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:15:49,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:51,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:52,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:15:52,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:15:52,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:15:52,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:15:53,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:55,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:57,603 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:15:57,637 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:15:57,639 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:15:57,639 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:15:57,640 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:15:57,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:57,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:15:58,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:15:59,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:01,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:03,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:16:03,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:05,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:07,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:16:07,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:16:07,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:16:07,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:08,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:16:09,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:11,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:13,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:16:13,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:15,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:17,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:18,790 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:16:19,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:16:19,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:22,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:22,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:16:22,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:16:22,993 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:16:24,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:24,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:16:26,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:28,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:30,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:30,233 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:16:32,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:34,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:35,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:16:36,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:37,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:16:37,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:16:37,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:16:38,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:40,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:41,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:16:42,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:43,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:16:43,024 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:16:43,024 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:16:43,027 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:16:44,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:44,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:16:45,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:46,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:46,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:16:48,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:48,805 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:16:50,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:52,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:52,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:16:52,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:16:52,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:16:52,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:16:54,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:56,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:16:57,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:16:58,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:00,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:02,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:02,932 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:17:04,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:06,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:07,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:17:07,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:17:07,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:17:08,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:08,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:17:10,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:12,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:13,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:17:14,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:16,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:18,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:18,794 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:17:19,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:17:20,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:22,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:22,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:17:23,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:17:23,020 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:17:24,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:24,734 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:17:26,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:28,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:28,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:17:28,389 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:17:28,390 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:17:28,392 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:17:29,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:17:30,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:30,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:17:31,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:32,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:34,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:35,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:17:36,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:37,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:17:38,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:17:38,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:17:38,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:40,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:41,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:17:42,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:44,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:46,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:46,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:17:48,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:48,796 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:17:50,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:52,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:17:52,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:52,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:17:53,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:17:53,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:17:54,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:56,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:17:57,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:17:58,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:00,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:02,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:02,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:18:04,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:06,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:07,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:18:08,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:18:08,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:18:08,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:18:08,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:10,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:12,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:13,782 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:18:13,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:18:13,815 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:18:13,815 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:18:13,817 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:18:14,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:14,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:18:15,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:16,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:18,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:18,798 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:18:18,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:18:20,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:22,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:22,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:18:23,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:18:23,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:18:24,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:24,143 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:18:26,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:28,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:29,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:18:30,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:32,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:34,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:34,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:18:36,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:37,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:18:38,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:18:38,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:18:38,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:40,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:40,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:18:42,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:44,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:45,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:18:46,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:48,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:48,800 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:18:50,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:51,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:18:52,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:52,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:18:53,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:18:53,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:18:54,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:56,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:56,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:18:58,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:18:59,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:18:59,146 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:18:59,147 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:18:59,148 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:18:59,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:19:00,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:01,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:01,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:19:02,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:04,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:06,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:07,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:19:07,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:19:08,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:19:08,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:19:08,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:10,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:12,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:12,760 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:19:14,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:16,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:18,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:18,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:19:18,803 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:19:20,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:22,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:22,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:19:23,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:19:23,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:19:24,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:19:24,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:26,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:28,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:29,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:19:30,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:32,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:34,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:34,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:19:36,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:37,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:19:38,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:19:38,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:19:38,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:40,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:19:40,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:42,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:44,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:44,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:19:44,462 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:19:44,462 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:19:44,464 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:19:45,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:19:45,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:19:46,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:47,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:48,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:48,805 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:19:50,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:50,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:19:52,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:52,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:19:53,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:19:53,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:19:54,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:56,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:19:56,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:19:58,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:00,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:01,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:20:02,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:04,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:06,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:07,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:20:07,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:20:08,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:20:08,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:20:08,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:10,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:12,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:12,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:20:14,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:16,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:17,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:20:18,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:18,806 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:20:20,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:22,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:22,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:20:23,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:20:23,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:20:23,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:20:24,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:26,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:28,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:28,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:20:29,662 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:20:29,663 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:20:29,663 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:20:29,665 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:20:30,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:30,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:20:31,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:32,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:34,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:34,256 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:20:36,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:37,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:20:38,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:20:38,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:20:38,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:39,675 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:20:40,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:42,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:44,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:45,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:20:46,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:48,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:48,809 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:20:50,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:50,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:20:52,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:52,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:20:53,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:20:53,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:20:54,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:55,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:20:56,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:20:58,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:00,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:01,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:21:02,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:04,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:06,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:06,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:21:07,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:21:08,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:21:08,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:21:08,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:10,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:12,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:21:12,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:14,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:14,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:21:14,833 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:21:14,833 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:21:14,835 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:21:15,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:21:16,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:17,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:17,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:21:18,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:18,811 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:21:20,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:22,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:22,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:21:22,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:21:23,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:21:23,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:21:24,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:26,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:28,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:28,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:21:30,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:32,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:33,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:21:34,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:36,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:37,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:21:38,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:21:38,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:21:38,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:39,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:21:40,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:42,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:44,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:44,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:21:46,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:48,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:48,814 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:21:50,141 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:21:50,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:52,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:52,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:21:53,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:21:53,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:21:54,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:55,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:21:56,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:21:58,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:00,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:22:00,024 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:22:00,025 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:22:00,027 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:22:00,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:00,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:22:01,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:22:02,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:03,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:04,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:06,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:06,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:22:07,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:22:08,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:22:08,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:22:08,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:10,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:11,831 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:22:12,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:14,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:16,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:17,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:22:18,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:18,817 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:22:20,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:22,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:22,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:22:22,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:22:23,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:22:23,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:22:24,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:26,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:28,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:22:28,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:30,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:32,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:33,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:22:34,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:36,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:37,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:22:38,045 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:22:38,046 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:22:38,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:39,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:22:40,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:42,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:44,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:22:44,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:45,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:22:45,221 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:22:45,222 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:22:45,223 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:22:45,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:22:46,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:48,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:48,819 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:22:49,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:22:50,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:52,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:52,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:22:53,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:22:53,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:22:54,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:55,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:22:56,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:22:58,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:00,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:00,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:23:02,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:04,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:06,043 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:23:06,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:07,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:23:08,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:23:08,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:23:08,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:10,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:11,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:23:12,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:14,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:16,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:16,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:23:18,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:18,834 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:23:20,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:22,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:23:22,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:22,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:23:23,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:23:23,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:23:24,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:26,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:27,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:23:28,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:30,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:23:30,386 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:23:30,386 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:23:30,388 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:23:30,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:30,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:23:32,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:33,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:23:34,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:36,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:37,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:23:38,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:23:38,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:23:38,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:39,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:23:40,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:42,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:44,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:44,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:23:46,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:48,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:48,823 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:23:49,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:23:50,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:52,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:52,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:23:53,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:23:53,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:23:54,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:55,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:23:56,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:23:58,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:00,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:00,961 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:24:02,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:04,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:06,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:24:06,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:07,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:24:08,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:24:08,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:24:08,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:10,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:11,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:24:12,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:14,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:15,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:24:15,486 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:24:15,486 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:24:15,487 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:24:16,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:16,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:24:17,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:24:18,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:18,825 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:24:20,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:22,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:22,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:24:22,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:24:23,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:24:23,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:24:24,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:26,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:27,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:28,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:24:29,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:31,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:33,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:24:33,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:35,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:37,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:37,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:24:38,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:24:38,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:24:38,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:24:39,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:41,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:43,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:44,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:24:45,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:47,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:48,828 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:24:49,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:49,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:24:51,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:52,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:24:53,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:24:53,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:24:53,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:55,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:24:55,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:57,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:24:59,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:00,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:25:00,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:25:00,878 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:25:00,879 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:25:00,880 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:25:01,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:01,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:25:02,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:03,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:05,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:06,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:25:07,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:07,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:25:08,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:25:08,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:25:09,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:11,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:11,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:25:13,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:15,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:17,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:25:17,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:18,842 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:25:19,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:21,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:22,558 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:25:22,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:25:23,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:25:23,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:25:23,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:25,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:27,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:27,975 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:25:29,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:31,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:33,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:25:33,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:35,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:37,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:37,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:25:38,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:25:38,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:25:38,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:25:39,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:41,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:43,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:44,356 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:25:45,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:46,202 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:25:46,203 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:25:46,204 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:25:46,205 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:25:46,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:25:47,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:48,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:48,834 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:25:49,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:49,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:25:51,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:52,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:25:53,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:25:53,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:25:53,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:55,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:25:55,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:57,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:25:59,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:00,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:26:01,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:03,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:05,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:06,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:26:07,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:07,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:26:08,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:26:08,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:26:09,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:11,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:26:11,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:13,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:15,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:16,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:26:17,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:18,836 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:26:19,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:21,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:22,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:26:22,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:26:23,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:26:23,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:26:23,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:25,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:27,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:26:27,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:29,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:31,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:26:31,438 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:26:31,438 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:26:31,440 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:26:31,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:31,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:26:32,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:33,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:26:33,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:35,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:37,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:37,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:26:38,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:26:38,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:26:38,658 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:26:39,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:41,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:43,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:44,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:26:45,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:47,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:48,838 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:26:49,512 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:26:49,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:51,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:52,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:26:53,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:26:53,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:26:53,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:54,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:26:55,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:57,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:26:59,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:00,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:27:01,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:03,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:05,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:27:05,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:07,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:07,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:27:08,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:27:08,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:27:09,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:11,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:27:11,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:13,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:15,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:16,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:27:16,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:27:16,584 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:27:16,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:27:16,588 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:27:16,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:27:17,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:18,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:18,859 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:27:19,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:21,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:21,976 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:27:22,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:27:23,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:27:23,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:27:23,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:25,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:27,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:27:27,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:29,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:31,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:32,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:27:33,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:35,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:37,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:37,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:27:38,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:27:38,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:27:38,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:27:39,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:41,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:43,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:27:43,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:45,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:47,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:48,842 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:27:48,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:27:49,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:51,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:52,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:27:53,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:27:53,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:27:53,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:54,117 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:27:55,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:57,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:27:59,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:27:59,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:01,795 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:28:01,797 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:28:01,797 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:28:01,799 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:28:01,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:01,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:28:02,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:03,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:04,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:28:05,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:07,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:28:07,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:08,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:28:08,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:28:09,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:10,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:28:11,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:13,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:15,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:28:15,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:17,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:18,844 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:28:19,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:20,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:28:21,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:22,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:28:23,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:28:23,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:28:23,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:25,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:26,275 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:28:27,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:29,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:31,683 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:28:31,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:33,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:35,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:37,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:28:37,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:28:37,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:38,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:28:38,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:28:39,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:41,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:42,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:28:43,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:45,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:46,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:28:46,968 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:28:46,968 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:28:46,970 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:28:47,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:47,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:28:47,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:28:48,846 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:28:48,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:49,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:51,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:52,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:28:53,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:28:53,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:28:53,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:28:53,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:55,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:57,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:28:58,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:28:59,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:01,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:03,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:04,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:29:05,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:07,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:29:07,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:08,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:29:08,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:29:09,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:29:09,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:11,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:13,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:14,987 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:29:15,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:17,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:18,849 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:29:19,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:20,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:29:21,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:22,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:29:23,107 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:29:23,107 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:29:23,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:25,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:29:25,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:27,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:29,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:31,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:29:31,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:32,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:29:32,056 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:29:32,057 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:29:32,058 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:29:32,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:29:33,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:34,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:35,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:36,642 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:29:37,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:29:37,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:38,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:29:38,087 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:29:39,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:41,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:42,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:29:43,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:45,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:47,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:29:47,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:48,851 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:29:49,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:51,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:52,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:29:52,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:29:53,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:29:53,087 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:29:54,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:56,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:58,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:29:58,255 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:30:00,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:02,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:03,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:30:04,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:06,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:07,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:30:08,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:08,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:30:08,161 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:30:09,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:30:10,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:12,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:14,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:14,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:30:16,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:17,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:30:17,109 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:30:17,110 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:30:17,111 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:30:18,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:18,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:30:18,853 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:30:19,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:19,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:30:20,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:22,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:22,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:30:23,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:30:23,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:30:24,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:25,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:30:26,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:28,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:30,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:30,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:30:32,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:34,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:36,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:36,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:30:37,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:30:38,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:38,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:30:38,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:30:40,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:41,559 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:30:42,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:44,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:46,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:46,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:30:48,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:48,855 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:30:50,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:52,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:52,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:30:52,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:30:53,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:30:53,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:30:54,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:56,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:30:57,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:30:58,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:00,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:02,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:02,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:31:02,248 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:31:02,248 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:31:02,249 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:31:03,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:31:03,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:31:04,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:05,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:06,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:07,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:31:08,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:08,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:31:08,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:31:09,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:31:10,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:12,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:14,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:14,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:31:16,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:18,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:18,857 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:31:20,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:20,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:31:22,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:22,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:31:23,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:31:23,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:31:24,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:25,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:31:26,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:28,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:30,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:31,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:31:32,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:34,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:36,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:36,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:31:37,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:31:38,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:31:38,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:31:38,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:40,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:41,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:31:42,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:44,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:46,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:47,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:31:47,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:31:47,393 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:31:47,393 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:31:47,395 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:31:48,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:48,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:31:48,859 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:31:49,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:50,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:52,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:52,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:31:52,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:31:53,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:31:53,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:31:54,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:56,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:58,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:31:58,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:32:00,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:02,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:03,568 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:32:04,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:06,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:07,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:32:08,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:32:08,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:32:08,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:08,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:32:10,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:12,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:14,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:14,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:32:16,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:18,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:18,862 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:32:19,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:32:20,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:22,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:22,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:32:23,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:32:23,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:32:24,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:25,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:32:26,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:28,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:30,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:30,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:32:32,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:32,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:32:32,442 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:32:32,442 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:32:32,443 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:32:33,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:32:34,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:35,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:36,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:32:36,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:37,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:32:38,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:32:38,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:32:38,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:40,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:41,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:32:42,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:44,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:46,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:46,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:32:48,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:48,864 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:32:50,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:52,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:52,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:32:52,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:32:53,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:32:53,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:32:54,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:56,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:32:57,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:32:58,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:00,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:02,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:03,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:33:04,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:06,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:07,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:33:08,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:33:08,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:33:08,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:08,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:33:10,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:12,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:13,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:33:14,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:16,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:17,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:33:17,543 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:33:17,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:33:17,545 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:33:18,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:18,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:33:18,866 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:33:19,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:19,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:33:20,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:22,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:22,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:33:23,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:33:23,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:33:24,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:24,711 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:33:26,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:28,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:30,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:33:30,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:32,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:34,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:35,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:33:36,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:37,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:33:38,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:33:38,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:33:38,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:40,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:40,941 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:33:42,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:44,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:46,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:46,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:33:48,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:48,897 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:33:50,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:51,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:33:52,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:52,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:33:53,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:33:53,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:33:54,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:56,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:33:57,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:33:58,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:00,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:02,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:02,683 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:34:02,715 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:34:02,716 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:34:02,716 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:34:02,718 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:34:03,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:34:04,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:05,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:06,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:07,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:34:07,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:34:08,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:34:08,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:34:08,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:10,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:12,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:13,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:34:14,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:16,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:18,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:18,870 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:34:18,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:34:20,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:22,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:22,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:34:23,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:34:23,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:34:24,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:34:24,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:26,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:28,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:29,854 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:34:30,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:32,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:34,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:35,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:34:36,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:37,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:34:38,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:34:38,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:34:38,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:40,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:40,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:34:42,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:44,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:46,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:34:46,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:48,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:34:48,002 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:34:48,002 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:34:48,004 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:34:48,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:48,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:34:48,872 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:34:49,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:50,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:51,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:34:52,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:52,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:34:53,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:34:53,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:34:54,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:56,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:34:56,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:34:58,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:00,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:02,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:02,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:35:04,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:06,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:07,818 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:35:07,932 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:35:08,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:35:08,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:35:08,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:10,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:12,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:13,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:35:14,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:16,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:18,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:18,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:35:18,874 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:35:20,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:22,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:22,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:35:23,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:35:23,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:35:24,156 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:35:24,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:26,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:28,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:29,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:35:30,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:32,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:33,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:35:33,297 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:35:33,298 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:35:33,299 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:35:33,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:35:34,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:35,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:35:36,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:37,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:35:38,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:35:38,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:35:38,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:40,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:35:40,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:42,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:44,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:45,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:35:46,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:48,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:48,889 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:35:50,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:51,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:35:52,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:52,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:35:53,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:35:53,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:35:54,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:56,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:35:56,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:35:58,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:00,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:02,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:36:02,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:04,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:06,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:07,807 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:36:07,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:36:08,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:36:08,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:36:08,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:10,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:12,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:13,243 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:36:14,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:16,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:18,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:18,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:36:18,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:36:18,715 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:36:18,716 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:36:18,717 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:36:18,878 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:36:19,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:36:20,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:22,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:22,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:36:23,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:36:23,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:36:24,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:36:24,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:26,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:28,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:29,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:36:30,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:32,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:34,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:34,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:36:36,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:37,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:36:38,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:36:38,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:36:38,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:40,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:36:40,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:42,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:44,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:45,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:36:46,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:48,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:48,880 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:36:50,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:51,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:36:52,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:52,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:36:53,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:36:53,161 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:36:54,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:56,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:36:56,686 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:36:58,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:00,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:02,125 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:37:02,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:03,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:37:03,970 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:37:03,970 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:37:03,972 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:37:04,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:04,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:37:06,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:07,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:37:07,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:37:08,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:37:08,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:37:08,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:10,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:12,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:12,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:37:14,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:16,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:18,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:37:18,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:18,883 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:37:20,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:22,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:22,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:37:23,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:37:23,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:37:23,794 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:37:24,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:26,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:28,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:29,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:37:30,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:32,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:34,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:34,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:37:36,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:37,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:37:38,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:37:38,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:37:38,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:40,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:37:40,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:42,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:44,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:45,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:37:46,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:48,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:48,885 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:37:49,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:37:49,319 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:37:49,319 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:37:49,321 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:37:49,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:37:50,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:51,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:37:52,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:52,961 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:37:53,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:37:53,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:37:54,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:56,526 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:37:56,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:37:58,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:00,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:01,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:38:02,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:04,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:06,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:07,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:38:07,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:38:08,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:38:08,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:38:08,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:10,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:12,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:12,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:38:14,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:16,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:18,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:38:18,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:18,887 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:38:20,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:22,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:22,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:38:23,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:38:23,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:38:23,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:38:24,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:26,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:28,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:38:28,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:30,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:32,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:33,727 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:38:34,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:38:34,578 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:38:34,578 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:38:34,580 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:38:34,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:34,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:38:36,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:37,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:38:38,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:38:38,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:38:38,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:39,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:38:40,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:42,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:44,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:38:44,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:46,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:48,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:48,889 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:38:50,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:38:50,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:52,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:52,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:38:53,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:38:53,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:38:54,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:55,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:38:56,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:57,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:38:59,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:00,819 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:39:01,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:03,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:05,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:06,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:39:07,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:07,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:39:08,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:39:08,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:39:09,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:11,632 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:39:11,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:13,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:15,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:17,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:39:17,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:18,891 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:39:19,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:39:19,734 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:39:19,734 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:39:19,736 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:39:19,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:19,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:39:20,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:21,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:22,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:39:22,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:39:23,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:39:23,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:39:23,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:25,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:27,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:27,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:39:29,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:31,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:33,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:39:33,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:35,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:37,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:37,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:39:38,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:39:38,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:39:39,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:39:39,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:41,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:43,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:44,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:39:45,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:47,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:48,893 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:39:49,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:49,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:39:51,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:52,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:39:53,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:39:53,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:39:53,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:55,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:39:55,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:57,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:39:59,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:00,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:40:01,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:03,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:04,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:40:04,993 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:40:04,993 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:40:04,995 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:40:05,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:05,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:40:05,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:40:06,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:07,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:07,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:40:08,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:40:08,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:40:09,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:11,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:40:11,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:13,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:15,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:16,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:40:17,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:18,895 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:40:19,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:21,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:22,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:40:22,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:40:23,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:40:23,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:40:23,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:25,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:27,817 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:40:27,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:29,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:31,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:33,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:40:33,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:35,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:37,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:37,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:40:38,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:40:38,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:40:38,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:40:39,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:41,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:43,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:44,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:40:45,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:47,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:48,898 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:40:49,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:49,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:40:50,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:40:50,483 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:40:50,483 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:40:50,485 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:40:50,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:40:51,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:52,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:52,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:40:53,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:40:53,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:40:53,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:55,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:40:55,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:57,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:40:59,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:00,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:41:01,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:03,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:05,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:05,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:41:07,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:07,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:41:08,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:41:08,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:41:09,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:11,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:41:11,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:13,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:15,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:16,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:41:17,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:18,900 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:41:19,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:21,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:22,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:41:22,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:41:23,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:41:23,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:41:23,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:25,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:27,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:41:27,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:29,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:31,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:33,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:41:33,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:35,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:35,985 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:41:35,986 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:41:35,986 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:41:35,988 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:41:36,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:36,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:41:37,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:37,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:41:38,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:41:38,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:41:39,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:41:39,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:41,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:43,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:45,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:41:45,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:47,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:48,902 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:41:49,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:50,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:41:51,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:52,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:41:53,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:41:53,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:41:53,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:55,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:41:55,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:57,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:41:59,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:01,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:42:01,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:03,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:05,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:06,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:42:07,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:42:07,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:08,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:42:08,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:42:09,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:11,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:12,356 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:42:13,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:15,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:17,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:42:17,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:18,916 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:42:19,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:21,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:42:21,492 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:42:21,492 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:42:21,494 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:42:21,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:21,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:42:22,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:42:22,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:42:22,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:23,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:42:23,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:42:23,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:25,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:27,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:28,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:42:29,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:31,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:33,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:34,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:42:35,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:37,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:42:37,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:38,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:42:38,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:42:39,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:42:39,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:41,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:43,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:44,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:42:45,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:47,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:48,906 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:42:49,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:42:49,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:51,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:52,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:42:53,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:42:53,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:42:53,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:55,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:42:55,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:57,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:42:59,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:00,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:43:01,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:03,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:05,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:06,047 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:43:06,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:43:06,895 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:43:06,896 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:43:06,897 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:43:06,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:43:07,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:43:07,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:08,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:43:08,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:43:08,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:10,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:11,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:43:12,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:14,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:16,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:16,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:43:18,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:18,908 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:43:20,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:22,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:22,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:43:22,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:43:23,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:43:23,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:43:24,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:26,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:27,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:43:28,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:30,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:32,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:33,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:43:34,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:36,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:37,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:43:38,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:38,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:43:38,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:43:38,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:43:40,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:42,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:44,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:44,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:43:46,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:48,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:48,910 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:43:49,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:43:50,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:52,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:52,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:43:52,323 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:43:52,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:43:52,325 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:43:52,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:43:53,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:43:53,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:43:53,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:43:54,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:55,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:55,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:43:56,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:43:58,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:00,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:00,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:44:02,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:04,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:05,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:44:06,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:07,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:44:08,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:08,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:44:08,151 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:44:10,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:11,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:44:12,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:14,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:16,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:16,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:44:18,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:18,925 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:44:20,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:22,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:22,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:44:22,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:44:23,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:44:23,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:44:24,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:26,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:27,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:44:28,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:30,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:32,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:33,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:44:34,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:36,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:37,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:44:37,623 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:44:37,623 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:44:37,625 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:44:37,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:44:38,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:38,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:44:38,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:44:38,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:44:38,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:44:39,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:40,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:42,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:44,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:44:44,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:46,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:48,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:48,915 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:44:49,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:44:50,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:52,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:52,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:44:53,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:44:53,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:44:54,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:55,729 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:44:56,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:44:58,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:00,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:01,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:45:02,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:04,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:06,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:06,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:45:07,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:45:08,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:08,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:45:08,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:45:10,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:12,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:12,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:45:14,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:16,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:17,762 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:45:18,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:18,917 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:45:20,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:22,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:22,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:45:22,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:45:23,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:45:23,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:45:23,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:45:23,508 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:45:23,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:45:23,510 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:45:24,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:24,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:45:25,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:26,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:28,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:28,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:45:30,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:32,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:34,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:45:34,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:36,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:37,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:45:38,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:45:38,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:45:38,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:39,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:45:40,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:42,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:44,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:45,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:45:46,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:48,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:48,919 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:45:50,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:50,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:45:52,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:52,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:45:53,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:45:53,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:45:54,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:55,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:45:56,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:45:58,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:00,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:00,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:46:02,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:04,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:06,123 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:46:06,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:07,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:46:08,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:46:08,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:46:08,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:08,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:46:08,778 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:46:08,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:46:08,779 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:46:09,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:46:10,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:11,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:11,559 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:46:12,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:14,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:16,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:16,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:46:18,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:18,932 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:46:20,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:22,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:22,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:46:22,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:46:23,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:46:23,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:46:24,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:26,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:27,823 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:46:28,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:30,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:32,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:33,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:46:34,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:36,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:37,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:46:38,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:46:38,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:46:38,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:46:38,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:40,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:42,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:44,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:46:44,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:46,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:48,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:48,923 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:46:49,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:46:50,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:52,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:52,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:46:53,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:46:53,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:46:54,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:46:54,111 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:46:54,112 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:46:54,113 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:46:54,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:54,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:46:55,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:46:55,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:56,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:46:58,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:00,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:00,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:47:02,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:04,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:05,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:47:06,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:07,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:47:08,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:47:08,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:47:08,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:10,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:11,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:47:12,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:14,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:16,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:16,819 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:47:18,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:18,925 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:47:20,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:22,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:47:22,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:22,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:47:23,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:47:23,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:47:24,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:26,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:27,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:47:28,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:30,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:32,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:33,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:47:34,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:36,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:37,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:47:38,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:47:38,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:47:38,255 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:47:38,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:39,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:47:39,335 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:47:39,336 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:47:39,337 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:47:39,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:47:40,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:41,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:42,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:43,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:47:44,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:46,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:48,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:48,927 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:47:49,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:47:50,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:52,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:52,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:47:53,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:47:53,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:47:54,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:55,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:47:56,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:47:58,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:00,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:01,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:48:02,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:04,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:06,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:06,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:48:07,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:48:08,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:48:08,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:48:08,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:10,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:12,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:48:12,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:14,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:16,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:17,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:48:18,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:18,929 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:48:20,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:22,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:22,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:48:22,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:48:23,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:48:23,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:48:24,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:24,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:48:24,711 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:48:24,711 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:48:24,714 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:48:25,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:48:26,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:27,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:28,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:48:28,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:30,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:32,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:33,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:48:34,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:36,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:37,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:48:38,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:48:38,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:48:38,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:39,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:48:40,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:42,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:44,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:44,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:48:46,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:48,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:48,932 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:48:49,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:48:50,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:52,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:52,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:48:53,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:48:53,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:48:54,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:55,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:48:56,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:48:58,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:00,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:00,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:49:02,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:04,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:06,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:49:06,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:07,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:49:08,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:49:08,248 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:49:08,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:09,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:49:09,991 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:49:09,991 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:49:09,993 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:49:10,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:10,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:49:11,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:11,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:49:12,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:14,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:16,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:17,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:49:18,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:18,934 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:49:20,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:22,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:22,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:49:22,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:49:23,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:49:23,229 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:49:24,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:26,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:28,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:49:28,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:30,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:32,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:33,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:49:34,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:36,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:37,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:49:38,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:49:38,229 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:49:38,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:38,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:49:40,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:42,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:44,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:49:44,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:46,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:48,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:48,936 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:49:49,736 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:49:50,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:52,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:52,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:49:53,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:49:53,229 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:49:54,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:55,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:49:55,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:49:55,210 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:49:55,211 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:49:55,212 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:49:55,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:49:56,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:57,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:49:58,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:00,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:00,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:50:02,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:04,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:06,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:50:06,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:07,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:50:08,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:50:08,229 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:50:08,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:10,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:11,551 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:50:12,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:14,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:16,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:16,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:50:18,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:18,938 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:50:20,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:22,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:50:22,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:22,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:50:23,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:50:23,230 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:50:24,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:26,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:27,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:50:28,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:30,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:32,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:33,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:50:34,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:36,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:37,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:50:38,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:50:38,230 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:50:38,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:50:38,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:40,499 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:50:40,501 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:50:40,502 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:50:40,503 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:50:40,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:40,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:50:42,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:44,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:50:44,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:46,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:48,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:48,940 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:50:49,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:50:50,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:52,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:52,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:50:53,263 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:50:53,264 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:50:54,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:54,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:50:56,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:50:58,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:00,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:51:00,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:02,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:04,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:05,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:51:06,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:07,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:51:08,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:51:08,244 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:51:08,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:10,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:11,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:51:12,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:14,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:16,639 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:51:16,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:18,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:18,943 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:51:20,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:22,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:51:22,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:22,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:51:23,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:51:23,244 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:51:24,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:25,727 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:51:25,729 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:51:25,729 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:51:25,731 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:51:26,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:26,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:51:27,521 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:51:28,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:30,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:32,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:32,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:51:34,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:36,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:37,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:51:38,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:51:38,244 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:51:38,355 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:51:38,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:40,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:42,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:43,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:51:44,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:46,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:48,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:48,944 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:51:48,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:51:50,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:52,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:52,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:51:53,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:51:53,244 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:51:54,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:51:54,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:56,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:51:58,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:00,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:52:00,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:02,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:04,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:05,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:52:06,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:07,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:52:08,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:52:08,244 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:52:08,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:10,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:10,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:52:10,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:52:10,959 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:52:10,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:52:10,961 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:52:11,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:52:12,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:14,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:16,361 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:52:16,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:18,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:18,947 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:52:20,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:21,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:52:22,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:22,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:52:23,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:52:23,245 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:52:24,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:26,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:27,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:52:28,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:30,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:32,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:52:32,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:34,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:36,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:37,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:52:37,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:52:38,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:52:38,303 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:52:38,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:40,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:42,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:43,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:52:44,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:46,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:48,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:48,949 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:52:48,950 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:52:50,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:52,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:52,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:52:53,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:52:53,283 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:52:54,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:52:54,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:56,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:52:56,386 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:52:56,386 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:52:56,388 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:52:56,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:56,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:52:58,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:52:59,976 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:53:00,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:02,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:04,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:05,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:53:06,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:07,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:53:08,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:53:08,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:53:08,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:10,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:10,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:53:12,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:14,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:16,260 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:53:16,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:18,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:18,951 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:53:20,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:21,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:53:22,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:22,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:53:23,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:53:23,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:53:24,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:26,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:27,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:53:28,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:30,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:32,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:53:32,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:34,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:36,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:37,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:53:37,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:53:38,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:53:38,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:53:38,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:40,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:41,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:53:41,686 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:53:41,687 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:53:41,689 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:53:41,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:53:42,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:43,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:53:44,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:46,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:48,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:48,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:53:48,953 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:53:50,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:52,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:52,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:53:53,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:53:53,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:53:54,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:53:54,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:56,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:58,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:53:59,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:54:00,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:02,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:04,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:05,255 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:54:06,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:07,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:54:08,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:54:08,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:54:08,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:10,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:54:10,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:12,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:14,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:16,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:54:16,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:18,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:18,955 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:54:20,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:21,711 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:54:22,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:22,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:54:23,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:54:23,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:54:24,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:26,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:27,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:54:27,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:54:27,205 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:54:27,206 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:54:27,207 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:54:27,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:54:28,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:30,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:32,635 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:54:32,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:34,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:36,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:37,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:54:37,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:54:38,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:54:38,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:54:38,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:40,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:42,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:43,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:54:44,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:46,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:48,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:48,957 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:54:48,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:54:50,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:52,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:52,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:54:53,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:54:53,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:54:54,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:54:54,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:56,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:58,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:54:59,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:55:00,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:02,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:04,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:05,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:55:06,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:07,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:55:08,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:55:08,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:55:08,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:10,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:11,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:55:12,938 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:55:12,940 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:55:12,941 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:55:12,942 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:55:12,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:12,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:55:14,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:16,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:55:16,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:18,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:18,960 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:55:20,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:22,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:55:22,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:22,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:55:23,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:55:23,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:55:24,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:26,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:27,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:55:28,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:30,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:32,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:32,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:55:34,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:36,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:37,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:55:37,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:55:38,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:38,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:55:38,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:55:40,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:42,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:43,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:55:44,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:46,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:48,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:48,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:55:48,962 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:55:50,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:52,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:52,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:55:53,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:55:53,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:55:54,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:54,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:55:56,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:58,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:55:58,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:55:58,494 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:55:58,495 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:55:58,496 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:55:59,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:55:59,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:56:00,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:02,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:04,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:04,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:56:06,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:07,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:56:08,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:08,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:56:08,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:56:10,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:10,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:56:12,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:14,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:15,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:56:16,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:18,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:18,964 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:56:20,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:21,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:56:22,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:22,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:56:23,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:56:23,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:56:24,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:26,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:26,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:56:28,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:30,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:32,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:56:32,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:34,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:36,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:37,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:56:37,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:56:38,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:38,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:56:38,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:56:40,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:42,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:42,932 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:56:43,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:56:43,777 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:56:43,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:56:43,779 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:56:44,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:44,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:56:46,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:48,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:48,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:56:48,977 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:56:50,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:52,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:52,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:56:53,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:56:53,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:56:53,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:56:54,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:56,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:58,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:56:59,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:57:00,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:02,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:04,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:04,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:57:06,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:07,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:57:08,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:08,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:57:08,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:57:10,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:57:10,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:12,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:14,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:15,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:57:16,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:18,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:18,968 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:57:20,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:21,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:57:22,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:22,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:57:23,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:57:23,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:57:24,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:26,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:26,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:57:28,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:29,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:57:29,105 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:57:29,106 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:57:29,108 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:57:29,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:57:30,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:31,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:57:32,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:34,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:36,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:37,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:57:37,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:57:38,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:57:38,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:57:38,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:40,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:42,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:42,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:57:44,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:46,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:48,166 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:57:48,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:48,971 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:57:50,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:52,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:52,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:57:53,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:57:53,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:57:53,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:57:54,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:56,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:58,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:57:59,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:58:00,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:02,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:04,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:04,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:58:06,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:07,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:58:08,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:58:08,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:58:08,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:09,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:58:10,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:12,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:14,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:14,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:58:14,369 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:58:14,369 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:58:14,371 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:58:15,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:58:15,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:58:16,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:18,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:18,972 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:58:20,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:20,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:58:22,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:22,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:58:23,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:58:23,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:58:24,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:26,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:58:26,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:28,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:30,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:31,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:58:32,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:34,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:36,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:37,007 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:58:37,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:58:38,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:58:38,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:58:38,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:40,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:42,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:42,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:58:44,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:46,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:47,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:58:48,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:48,987 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:58:50,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:52,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:52,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:58:52,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:58:53,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:58:53,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:58:54,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:56,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:58,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:58:58,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:58:59,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:58:59,536 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:58:59,537 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:58:59,540 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:59:00,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:00,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:59:02,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:04,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:59:04,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:06,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:07,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:59:08,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:59:08,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:59:08,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:09,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:59:10,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:12,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:14,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:14,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:59:16,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:18,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:18,977 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:59:20,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:59:20,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:22,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:23,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:59:23,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:59:23,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:59:24,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:25,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:59:26,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:28,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:30,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:31,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:59:32,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:34,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:36,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:36,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:59:38,007 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:59:38,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:59:38,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:59:38,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:40,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:41,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:59:42,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:44,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:44,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 14:59:44,611 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 14:59:44,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 14:59:44,614 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 14:59:45,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 14:59:46,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:47,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:59:48,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:48,979 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 14:59:50,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:52,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:52,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:59:53,007 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 14:59:53,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 14:59:53,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 14:59:54,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:56,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 14:59:58,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 14:59:58,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:00,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:02,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:03,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:00:04,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:06,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:08,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:00:08,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:00:08,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:00:08,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:09,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:00:10,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:12,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:14,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:15,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:00:16,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:18,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:18,981 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:00:20,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:20,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:00:22,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:23,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:00:23,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:00:23,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:00:24,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:26,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:00:26,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:28,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:29,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:00:29,693 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:00:29,693 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:00:29,695 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:00:30,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:30,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:00:31,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:00:32,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:34,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:36,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:36,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:00:38,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:00:38,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:00:38,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:00:38,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:40,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:42,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:00:42,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:44,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:46,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:47,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:00:48,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:48,993 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:00:50,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:52,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:53,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:00:53,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:00:53,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:00:53,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:00:54,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:56,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:00:58,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:00:58,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:00,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:02,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:03,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:01:04,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:06,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:08,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:01:08,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:01:08,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:01:08,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:09,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:01:10,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:12,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:14,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:14,725 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:01:14,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:01:14,758 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:01:14,759 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:01:14,760 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:01:15,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:01:16,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:18,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:18,985 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:01:19,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:01:20,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:22,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:23,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:01:23,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:01:23,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:01:24,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:25,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:01:26,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:28,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:30,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:30,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:01:32,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:34,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:36,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:01:36,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:38,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:01:38,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:01:38,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:01:38,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:40,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:41,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:01:42,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:44,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:46,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:47,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:01:48,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:48,987 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:01:50,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:52,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:01:52,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:53,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:01:53,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:01:53,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:01:54,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:56,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:57,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:01:58,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:01:59,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:01:59,733 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:01:59,734 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:01:59,735 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:02:00,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:00,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:02:02,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:03,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:02:04,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:06,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:08,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:02:08,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:02:08,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:02:08,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:02:08,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:10,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:12,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:14,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:02:14,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:16,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:18,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:18,990 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:02:19,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:02:20,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:22,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:23,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:02:23,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:02:23,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:02:24,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:24,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:02:26,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:28,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:30,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:02:30,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:32,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:34,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:35,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:02:36,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:38,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:02:38,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:02:38,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:02:38,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:40,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:41,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:02:42,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:44,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:44,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:02:44,859 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:02:44,859 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:02:44,861 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:02:45,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:02:46,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:46,632 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:02:48,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:49,003 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:02:50,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:52,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:02:52,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:53,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:02:53,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:02:53,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:02:54,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:56,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:02:57,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:02:58,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:00,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:02,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:02,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:03:04,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:06,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:08,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:03:08,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:03:08,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:03:08,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:03:08,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:10,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:12,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:13,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:03:14,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:16,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:18,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:18,994 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:03:18,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:03:20,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:22,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:23,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:03:23,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:03:23,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:03:24,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:03:24,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:26,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:28,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:30,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:03:30,186 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:03:30,188 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:03:30,188 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:03:30,190 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:03:30,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:30,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:03:32,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:34,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:35,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:03:36,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:38,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:03:38,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:03:38,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:03:38,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:40,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:41,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:03:42,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:44,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:46,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:03:46,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:48,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:48,997 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:03:50,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:51,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:03:52,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:53,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:03:53,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:03:53,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:03:54,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:56,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:03:57,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:03:58,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:00,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:02,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:02,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:04:04,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:06,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:08,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:04:08,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:04:08,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:04:08,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:04:08,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:10,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:12,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:13,590 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:04:14,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:15,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:04:15,432 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:04:15,433 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:04:15,434 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:04:15,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:04:16,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:18,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:18,999 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:04:19,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:04:20,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:22,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:23,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:04:23,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:04:23,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:04:24,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:04:24,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:26,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:28,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:29,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:04:30,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:32,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:34,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:35,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:04:36,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:38,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:04:38,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:04:38,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:04:38,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:40,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:40,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:04:42,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:44,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:46,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:04:46,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:48,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:49,001 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:04:50,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:51,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:04:52,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:53,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:04:53,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:04:53,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:04:54,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:56,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:04:57,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:04:58,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:00,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:00,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:05:00,831 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:05:00,832 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:05:00,833 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:05:01,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:05:02,603 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:05:02,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:04,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:06,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:08,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:05:08,123 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:05:08,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:05:08,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:05:08,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:10,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:12,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:13,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:05:14,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:16,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:18,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:18,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:05:19,003 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:05:20,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:22,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:23,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:05:23,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:05:23,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:05:24,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:05:24,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:26,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:28,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:29,736 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:05:30,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:32,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:34,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:35,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:05:36,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:38,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:05:38,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:05:38,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:05:38,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:40,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:05:40,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:42,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:45,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:45,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:05:46,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:05:46,022 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:05:46,022 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:05:46,024 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:05:47,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:47,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:05:49,006 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:05:49,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:51,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:51,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:05:53,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:53,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:05:53,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:05:53,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:05:55,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:56,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:05:57,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:05:59,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:01,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:02,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:06:03,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:05,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:07,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:07,758 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:06:08,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:06:08,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:06:08,317 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:06:09,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:11,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:13,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:13,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:06:15,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:17,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:18,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:06:19,008 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:06:19,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:21,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:23,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:23,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:06:23,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:06:23,317 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:06:24,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:06:25,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:27,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:29,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:29,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:06:31,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:31,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:06:31,341 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:06:31,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:06:31,342 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:06:32,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:06:33,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:34,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:06:35,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:37,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:38,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:06:38,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:06:38,317 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:06:39,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:40,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:06:41,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:43,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:45,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:45,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:06:47,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:49,011 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:06:49,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:51,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:51,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:06:53,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:53,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:06:53,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:06:53,317 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:06:55,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:56,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:06:57,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:06:59,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:01,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:02,097 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:07:03,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:05,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:07,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:07,536 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:07:08,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:07:08,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:07:08,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:07:09,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:11,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:12,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:07:13,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:15,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:16,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:07:16,613 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:07:16,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:07:16,615 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:07:17,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:17,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:07:18,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:07:19,013 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:07:19,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:21,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:23,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:07:23,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:23,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:07:23,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:07:23,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:07:25,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:27,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:28,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:07:29,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:31,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:33,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:33,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:07:35,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:37,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:38,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:07:38,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:07:38,337 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:07:39,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:39,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:07:41,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:43,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:44,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:07:45,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:47,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:49,015 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:07:49,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:50,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:07:51,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:53,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:07:53,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:53,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:07:53,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:07:55,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:55,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:07:57,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:07:59,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:01,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:08:01,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:02,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:08:02,031 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:08:02,031 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:08:02,033 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:08:02,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:08:03,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:05,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:06,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:08:07,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:08,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:08:08,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:08:08,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:08:09,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:11,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:12,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:08:13,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:15,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:17,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:17,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:08:19,017 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:08:19,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:21,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:22,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:08:23,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:08:23,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:23,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:08:23,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:08:25,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:27,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:28,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:08:29,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:31,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:33,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:33,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:08:35,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:37,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:38,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:08:38,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:08:38,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:08:39,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:39,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:08:41,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:43,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:44,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:08:45,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:47,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:47,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:08:47,437 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:08:47,438 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:08:47,439 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:08:48,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:08:49,019 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:08:49,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:50,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:08:51,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:53,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:08:53,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:53,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:08:53,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:08:55,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:55,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:08:57,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:08:59,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:01,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:09:01,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:03,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:05,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:06,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:09:07,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:08,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:09:08,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:09:08,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:09:09,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:11,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:12,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:09:13,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:15,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:17,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:17,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:09:19,034 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:09:19,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:21,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:23,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:09:23,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:09:23,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:23,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:09:23,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:09:25,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:27,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:28,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:09:29,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:31,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:33,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:09:33,014 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:09:33,015 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:09:33,018 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:09:33,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:33,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:09:34,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:09:35,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:37,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:38,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:09:38,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:09:38,319 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:09:39,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:39,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:09:41,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:43,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:44,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:09:45,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:47,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:49,024 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:09:49,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:50,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:09:51,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:53,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:09:53,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:53,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:09:53,319 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:09:55,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:55,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:09:57,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:09:59,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:01,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:10:01,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:03,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:05,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:06,683 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:10:07,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:08,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:10:08,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:10:08,319 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:10:09,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:11,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:12,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:10:13,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:15,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:17,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:17,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:10:18,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:10:18,399 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:10:18,400 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:10:18,401 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:10:19,026 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:10:19,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:19,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:10:21,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:22,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:10:23,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:10:23,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:23,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:10:23,319 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:10:25,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:27,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:28,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:10:29,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:31,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:33,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:33,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:10:35,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:37,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:38,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:10:38,363 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:10:38,364 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:10:39,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:39,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:10:41,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:42,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:44,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:44,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:10:46,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:48,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:49,028 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:10:50,139 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:10:50,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:52,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:53,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:10:53,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:10:53,344 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:10:54,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:55,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:10:56,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:10:58,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:00,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:01,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:11:02,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:03,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:11:03,686 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:11:03,686 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:11:03,688 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:11:04,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:04,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:11:06,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:06,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:11:08,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:11:08,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:11:08,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:11:08,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:10,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:11,906 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:11:12,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:14,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:16,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:17,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:11:18,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:19,041 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:11:20,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:22,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:22,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:11:23,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:11:23,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:11:23,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:11:24,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:26,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:28,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:11:28,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:30,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:32,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:33,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:11:34,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:36,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:38,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:11:38,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:11:38,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:11:38,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:39,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:11:40,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:42,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:44,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:44,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:11:46,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:48,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:48,898 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:11:48,900 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:11:48,900 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:11:48,902 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:11:49,033 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:11:49,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:11:50,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:11:50,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:52,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:53,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:11:53,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:11:53,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:11:54,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:55,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:11:56,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:11:58,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:00,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:00,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:12:02,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:04,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:06,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:12:06,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:08,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:12:08,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:12:08,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:12:08,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:10,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:11,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:12:12,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:14,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:16,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:17,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:12:18,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:19,035 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:12:20,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:22,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:22,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:12:23,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:12:23,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:12:23,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:12:24,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:26,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:27,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:12:28,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:30,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:32,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:33,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:12:34,107 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:12:34,109 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:12:34,109 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:12:34,111 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:12:34,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:34,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:12:36,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:38,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:12:38,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:12:38,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:12:38,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:12:38,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:40,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:42,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:44,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:12:44,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:46,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:48,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:49,037 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:12:50,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:12:50,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:52,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:53,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:12:53,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:12:53,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:12:54,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:55,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:12:56,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:12:58,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:00,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:01,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:13:02,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:04,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:06,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:06,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:13:08,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:13:08,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:13:08,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:13:08,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:10,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:11,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:13:12,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:14,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:16,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:17,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:13:18,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:19,052 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:13:19,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:13:19,245 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:13:19,246 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:13:19,247 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:13:19,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:13:20,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:22,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:22,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:13:23,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:13:23,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:13:23,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:13:24,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:26,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:28,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:13:28,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:30,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:32,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:33,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:13:34,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:36,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:38,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:13:38,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:13:38,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:13:38,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:39,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:13:40,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:42,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:44,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:13:44,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:46,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:48,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:49,042 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:13:49,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:13:50,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:52,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:53,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:13:53,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:13:53,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:13:54,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:55,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:13:56,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:13:58,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:00,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:00,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:14:02,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:04,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:14:04,270 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:14:04,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:14:04,272 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:14:04,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:04,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:14:06,046 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:14:06,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:08,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:14:08,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:14:08,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:14:08,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:10,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:11,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:14:12,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:14,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:16,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:16,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:14:18,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:19,044 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:14:20,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:22,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:14:22,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:23,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:14:23,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:14:23,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:14:24,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:26,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:27,730 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:14:28,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:30,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:32,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:33,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:14:34,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:36,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:38,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:14:38,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:14:38,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:14:38,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:14:38,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:40,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:42,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:43,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:14:44,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:46,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:48,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:49,046 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:14:49,047 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:14:49,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:14:49,347 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:14:49,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:14:49,349 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:14:49,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:14:50,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:52,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:53,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:14:53,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:14:53,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:14:54,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:14:54,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:56,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:14:58,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:00,125 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:15:00,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:02,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:04,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:05,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:15:06,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:08,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:15:08,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:15:08,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:15:08,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:10,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:10,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:15:12,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:14,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:16,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:15:16,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:18,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:19,059 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:15:20,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:21,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:15:22,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:23,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:15:23,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:15:23,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:15:24,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:26,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:27,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:15:28,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:30,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:32,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:15:32,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:34,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:15:34,383 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:15:34,383 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:15:34,385 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:15:34,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:34,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:15:36,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:37,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:15:38,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:15:38,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:15:38,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:15:38,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:40,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:42,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:43,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:15:44,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:46,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:48,810 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:15:48,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:49,051 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:15:50,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:52,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:53,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:15:53,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:15:53,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:15:54,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:15:54,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:56,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:58,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:15:59,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:16:00,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:02,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:04,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:05,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:16:06,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:08,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:16:08,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:16:08,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:16:08,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:10,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:16:10,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:12,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:14,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:15,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:16:16,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:18,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:19,054 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:16:19,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:16:19,450 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:16:19,450 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:16:19,452 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:16:19,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:16:20,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:21,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:16:22,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:23,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:16:23,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:16:23,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:16:24,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:26,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:16:26,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:28,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:30,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:32,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:16:32,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:34,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:36,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:37,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:16:38,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:16:38,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:16:38,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:16:38,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:40,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:42,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:16:42,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:44,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:46,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:48,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:16:48,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:49,056 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:16:50,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:52,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:53,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:16:53,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:16:53,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:16:53,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:16:54,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:56,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:58,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:16:59,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:17:00,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:02,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:04,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:17:04,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:17:04,427 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:17:04,428 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:17:04,430 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:17:04,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:04,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:17:06,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:08,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:17:08,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:17:08,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:17:08,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:09,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:17:10,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:12,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:14,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:15,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:17:16,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:19,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:19,058 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:17:20,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:17:21,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:23,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:23,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:17:23,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:17:23,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:17:25,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:26,109 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:17:27,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:29,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:31,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:31,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:17:33,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:35,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:36,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:17:37,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:38,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:17:38,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:17:38,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:17:39,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:41,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:42,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:17:43,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:45,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:47,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:47,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:17:49,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:49,060 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:17:49,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:17:49,492 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:17:49,493 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:17:49,494 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:17:50,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:17:51,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:53,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:53,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:17:53,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:17:53,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:17:53,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:17:55,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:57,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:17:58,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:17:59,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:01,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:03,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:03,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:18:05,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:07,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:08,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:18:08,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:18:08,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:18:09,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:09,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:18:11,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:13,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:14,631 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:18:15,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:17,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:19,063 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:18:19,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:20,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:18:21,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:23,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:23,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:18:23,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:18:23,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:18:25,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:25,429 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:18:27,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:29,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:30,808 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:18:31,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:33,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:34,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:18:34,426 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:18:34,426 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:18:34,428 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:18:35,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:35,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:18:36,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:18:37,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:38,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:18:38,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:18:38,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:18:39,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:41,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:41,559 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:18:43,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:45,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:46,931 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:18:47,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:49,065 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:18:49,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:51,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:52,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:18:53,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:53,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:18:53,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:18:53,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:18:55,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:57,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:18:57,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:18:59,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:01,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:03,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:03,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:19:05,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:07,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:08,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:19:08,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:19:08,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:19:08,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:19:09,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:11,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:13,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:13,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:19:15,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:17,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:19,067 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:19:19,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:19:19,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:19,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:19:19,351 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:19:19,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:19:19,353 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:19:20,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:19:21,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:23,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:19:23,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:23,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:19:23,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:19:24,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:19:25,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:27,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:29,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:30,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:19:31,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:33,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:35,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:35,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:19:37,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:38,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:19:38,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:19:38,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:19:39,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:40,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:19:41,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:43,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:45,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:46,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:19:47,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:49,070 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:19:49,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:51,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:51,683 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:19:53,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:19:53,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:53,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:19:53,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:19:55,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:57,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:19:57,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:19:59,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:01,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:02,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:20:03,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:04,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:20:04,290 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:20:04,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:20:04,293 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:20:05,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:05,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:20:07,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:07,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:20:08,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:20:08,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:20:08,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:20:09,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:11,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:13,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:13,275 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:20:15,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:17,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:18,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:20:19,072 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:20:19,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:21,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:23,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:20:23,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:23,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:20:23,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:20:24,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:20:25,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:27,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:29,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:29,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:20:31,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:33,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:34,788 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:20:35,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:37,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:38,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:20:38,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:20:38,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:20:39,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:40,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:20:41,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:43,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:45,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:45,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:20:47,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:49,074 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:20:49,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:20:49,178 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:20:49,179 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:20:49,181 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:20:49,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:49,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:20:50,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:20:51,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:53,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:20:53,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:53,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:20:53,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:20:55,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:56,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:20:57,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:20:59,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:01,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:01,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:21:03,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:05,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:07,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:21:07,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:08,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:21:08,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:21:08,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:21:09,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:11,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:12,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:21:13,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:15,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:17,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:17,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:21:19,076 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:21:19,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:21,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:23,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:21:23,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:21:23,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:23,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:21:23,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:21:25,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:27,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:28,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:21:29,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:31,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:33,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:33,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:21:34,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:21:34,005 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:21:34,005 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:21:34,007 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:21:34,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:21:35,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:37,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:38,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:21:38,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:21:38,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:21:39,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:39,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:21:41,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:43,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:44,807 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:21:45,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:47,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:49,079 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:21:49,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:50,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:21:51,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:53,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:21:53,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:53,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:21:53,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:21:55,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:55,592 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:21:57,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:21:59,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:00,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:22:01,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:03,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:05,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:06,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:22:07,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:08,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:22:08,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:22:08,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:22:09,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:11,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:11,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:22:13,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:15,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:17,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:22:17,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:18,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:22:18,992 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:22:18,992 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:22:18,994 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:22:19,081 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:22:19,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:19,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:22:21,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:22,559 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:22:23,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:22:23,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:23,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:22:23,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:22:25,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:27,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:27,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:22:29,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:31,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:33,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:22:33,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:35,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:37,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:38,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:22:38,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:22:38,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:22:38,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:22:39,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:41,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:43,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:44,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:22:45,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:47,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:49,083 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:22:49,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:22:49,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:51,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:53,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:22:53,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:22:53,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:22:53,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:54,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:22:55,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:57,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:22:59,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:00,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:23:01,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:03,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:03,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:23:03,914 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:23:03,914 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:23:03,916 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:23:04,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:23:05,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:05,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:23:07,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:08,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:23:08,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:23:08,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:23:09,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:11,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:23:11,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:13,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:15,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:16,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:23:17,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:19,085 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:23:19,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:21,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:21,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:23:23,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:23:23,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:23:23,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:23:23,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:25,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:27,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:23:27,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:29,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:31,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:32,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:23:33,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:35,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:37,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:37,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:23:38,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:23:38,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:23:38,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:23:39,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:41,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:43,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:23:43,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:45,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:47,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:48,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:23:48,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:23:48,785 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:23:48,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:23:48,787 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:23:49,088 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:23:49,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:49,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:23:51,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:53,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:23:53,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:23:53,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:23:53,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:54,109 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:23:55,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:57,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:59,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:23:59,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:24:01,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:03,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:04,851 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:24:05,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:07,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:08,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:24:08,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:24:08,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:24:09,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:10,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:24:11,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:13,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:15,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:15,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:24:17,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:19,157 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:24:19,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:20,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:24:21,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:23,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:24:23,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:24:23,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:24:23,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:25,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:26,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:24:27,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:29,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:31,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:31,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:24:33,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:33,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:24:33,566 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:24:33,566 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:24:33,568 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:24:34,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:24:35,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:37,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:24:37,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:38,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:24:38,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:24:38,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:24:39,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:41,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:42,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:24:43,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:45,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:47,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:47,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:24:49,160 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:24:49,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:51,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:53,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:24:53,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:24:53,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:24:53,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:24:53,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:55,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:57,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:24:58,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:24:59,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:01,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:03,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:03,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:25:05,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:07,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:08,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:25:08,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:25:08,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:25:09,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:25:09,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:11,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:13,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:14,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:25:15,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:17,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:18,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:25:18,394 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:25:18,394 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:25:18,396 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:25:18,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:25:19,162 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:25:19,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:20,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:25:21,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:23,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:25:23,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:25:23,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:25:23,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:25,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:25,605 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:25:27,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:29,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:31,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:25:31,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:33,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:35,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:36,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:25:37,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:38,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:25:38,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:25:38,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:25:39,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:41,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:41,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:25:43,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:45,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:47,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:25:47,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:49,174 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:25:49,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:51,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:52,558 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:25:53,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:25:53,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:25:53,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:25:53,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:55,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:57,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:25:57,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:25:59,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:01,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:03,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:26:03,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:26:03,387 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:26:03,387 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:26:03,389 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:26:03,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:03,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:26:05,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:07,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:08,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:26:08,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:26:08,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:26:08,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:26:09,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:11,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:12,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:14,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:26:14,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:16,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:18,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:19,166 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:26:19,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:26:20,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:22,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:23,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:26:23,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:26:23,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:26:24,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:25,007 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:26:26,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:28,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:30,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:26:30,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:32,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:34,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:35,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:26:36,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:38,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:26:38,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:26:38,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:26:38,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:40,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:41,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:26:42,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:44,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:46,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:26:46,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:48,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:26:48,324 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:26:48,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:26:48,326 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:26:48,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:48,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:26:49,168 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:26:50,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:51,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:26:52,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:53,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:26:53,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:26:53,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:26:54,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:56,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:26:57,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:26:58,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:00,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:02,622 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:27:02,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:04,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:06,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:08,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:27:08,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:27:08,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:27:08,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:27:08,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:10,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:12,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:13,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:27:14,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:16,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:18,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:18,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:27:19,171 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:27:20,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:22,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:23,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:27:23,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:27:23,352 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:27:23,811 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:27:24,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:26,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:28,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:29,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:27:30,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:32,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:33,156 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:27:33,159 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:27:33,159 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:27:33,160 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:27:33,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:27:34,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:34,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:27:36,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:38,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:27:38,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:27:38,352 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:27:38,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:40,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:27:40,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:42,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:44,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:45,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:27:46,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:48,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:49,183 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:27:50,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:50,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:27:52,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:53,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:27:53,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:27:53,352 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:27:54,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:56,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:27:56,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:27:58,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:00,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:01,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:28:02,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:04,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:06,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:07,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:28:08,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:28:08,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:28:08,352 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:28:08,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:10,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:12,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:28:12,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:14,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:16,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:17,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:28:17,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:28:17,915 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:28:17,916 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:28:17,917 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:28:18,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:18,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:28:19,175 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:28:20,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:22,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:23,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:28:23,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:28:23,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:28:23,352 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:28:24,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:26,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:28,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:28:28,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:30,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:32,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:33,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:28:34,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:36,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:38,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:28:38,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:28:38,352 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:28:38,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:39,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:28:40,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:42,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:44,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:44,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:28:46,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:48,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:49,177 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:28:50,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:28:50,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:52,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:53,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:28:53,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:28:53,352 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:28:54,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:55,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:28:56,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:28:58,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:00,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:00,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:29:02,690 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:29:02,692 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:29:02,693 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:29:02,695 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:29:02,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:02,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:29:04,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:06,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:29:06,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:08,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:29:08,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:29:08,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:29:08,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:10,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:11,598 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:29:12,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:14,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:16,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:16,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:29:18,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:19,179 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:29:20,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:22,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:29:22,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:23,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:29:23,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:29:23,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:29:24,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:26,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:27,810 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:29:28,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:30,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:32,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:33,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:29:34,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:36,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:38,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:29:38,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:29:38,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:29:38,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:29:38,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:40,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:42,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:43,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:29:44,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:46,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:47,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:29:47,603 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:29:47,604 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:29:47,605 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:29:47,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:29:48,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:49,192 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:29:49,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:29:50,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:52,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:53,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:29:53,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:29:53,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:29:54,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:29:54,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:56,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:29:58,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:00,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:30:00,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:02,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:04,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:05,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:30:06,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:08,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:30:08,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:30:08,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:30:08,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:10,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:10,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:30:12,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:14,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:16,243 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:30:16,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:18,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:19,184 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:30:20,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:21,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:30:22,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:23,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:30:23,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:30:23,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:30:24,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:26,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:27,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:30:28,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:30,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:32,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:30:32,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:30:32,457 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:30:32,458 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:30:32,460 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:30:32,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:32,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:30:34,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:36,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:37,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:30:38,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:30:38,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:30:38,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:30:38,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:40,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:42,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:43,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:30:44,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:46,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:48,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:30:48,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:49,186 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:30:50,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:52,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:53,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:30:53,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:30:53,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:30:53,961 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:30:54,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:56,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:58,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:30:59,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:31:00,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:02,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:04,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:31:04,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:06,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:08,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:31:08,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:31:08,386 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:31:08,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:10,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:31:10,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:12,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:14,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:15,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:31:16,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:17,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:31:17,312 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:31:17,312 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:31:17,314 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:31:17,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:31:18,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:19,188 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:31:20,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:31:20,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:22,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:23,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:31:23,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:31:23,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:31:24,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:26,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:31:26,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:28,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:30,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:31,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:31:32,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:34,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:36,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:36,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:31:38,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:31:38,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:31:38,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:31:38,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:40,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:42,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:31:42,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:44,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:46,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:47,690 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:31:48,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:49,191 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:31:50,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:52,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:53,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:31:53,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:31:53,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:31:53,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:31:54,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:56,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:31:58,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:31:58,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:00,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:02,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:32:02,084 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:32:02,085 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:32:02,086 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:32:02,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:02,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:32:03,853 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:32:04,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:06,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:08,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:32:08,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:32:08,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:32:08,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:09,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:32:10,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:12,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:14,627 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:32:14,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:16,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:18,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:19,193 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:32:20,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:32:20,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:22,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:23,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:32:23,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:32:23,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:32:24,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:25,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:32:26,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:28,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:30,823 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:32:30,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:32,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:34,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:36,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:32:36,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:38,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:32:38,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:32:38,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:32:38,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:40,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:41,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:32:43,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:45,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:46,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:32:46,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:32:46,994 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:32:46,994 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:32:46,996 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:32:47,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:47,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:32:49,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:49,195 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:32:51,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:52,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:32:53,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:53,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:32:53,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:32:53,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:32:55,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:57,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:32:57,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:32:59,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:01,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:03,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:03,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:33:05,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:07,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:08,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:33:08,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:33:08,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:33:08,536 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:33:09,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:11,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:13,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:13,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:33:15,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:17,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:19,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:19,198 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:33:19,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:33:21,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:23,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:23,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:33:23,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:33:23,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:33:24,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:33:25,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:27,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:29,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:30,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:33:31,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:31,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:33:31,909 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:33:31,910 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:33:31,911 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:33:32,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:33:33,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:35,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:35,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:33:37,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:38,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:33:38,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:33:38,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:33:39,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:40,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:33:41,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:43,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:45,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:46,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:33:47,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:49,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:49,200 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:33:51,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:51,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:33:53,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:53,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:33:53,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:33:53,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:33:55,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:56,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:33:57,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:33:59,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:01,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:02,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:34:03,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:05,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:07,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:07,701 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:34:08,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:34:08,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:34:08,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:34:09,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:11,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:13,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:34:13,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:15,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:16,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:34:16,687 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:34:16,687 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:34:16,689 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:34:17,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:17,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:34:18,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:34:19,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:19,202 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:34:21,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:23,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:23,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:34:23,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:34:23,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:34:23,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:34:25,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:27,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:29,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:29,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:34:31,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:33,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:34,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:34:35,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:37,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:38,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:34:38,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:34:38,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:34:39,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:39,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:34:41,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:43,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:45,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:45,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:34:47,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:49,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:49,204 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:34:50,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:34:51,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:53,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:53,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:34:53,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:34:53,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:34:55,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:56,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:34:57,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:34:59,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:01,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:01,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:35:01,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:35:01,477 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:35:01,478 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:35:01,479 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:35:02,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:35:03,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:05,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:06,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:35:07,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:08,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:35:08,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:35:08,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:35:09,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:11,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:12,185 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:35:13,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:15,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:17,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:17,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:35:19,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:19,207 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:35:21,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:22,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:35:23,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:35:23,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:23,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:35:23,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:35:25,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:27,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:28,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:35:29,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:31,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:33,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:33,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:35:35,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:37,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:38,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:35:38,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:35:38,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:35:39,047 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:35:39,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:41,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:43,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:44,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:35:45,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:46,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:35:46,243 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:35:46,243 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:35:46,245 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:35:46,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:35:47,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:49,209 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:35:49,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:49,794 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:35:51,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:53,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:35:53,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:35:53,409 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:35:53,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:55,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:35:55,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:57,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:35:59,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:00,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:36:01,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:03,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:05,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:05,919 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:36:07,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:08,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:36:08,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:36:08,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:36:09,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:11,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:36:11,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:13,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:15,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:16,738 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:36:17,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:19,213 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:36:19,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:21,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:22,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:36:23,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:36:23,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:36:23,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:36:23,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:25,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:27,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:36:27,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:29,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:31,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:36:31,092 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:36:31,093 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:36:31,094 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:36:31,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:31,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:36:32,854 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:36:33,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:35,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:37,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:38,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:36:38,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:36:38,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:36:38,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:36:39,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:41,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:43,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:43,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:36:45,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:47,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:48,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:36:49,214 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:36:49,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:51,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:53,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:36:53,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:36:53,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:36:53,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:54,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:36:55,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:57,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:59,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:36:59,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:37:01,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:03,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:05,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:37:05,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:07,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:08,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:37:08,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:37:08,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:37:09,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:10,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:37:11,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:13,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:15,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:15,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:37:15,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:37:15,906 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:37:15,906 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:37:15,908 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:37:16,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:37:17,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:19,217 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:37:19,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:21,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:37:21,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:23,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:37:23,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:37:23,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:37:23,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:25,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:26,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:37:27,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:29,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:31,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:31,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:37:33,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:35,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:37,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:37:37,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:38,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:37:38,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:37:38,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:37:39,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:41,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:42,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:37:43,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:45,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:47,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:48,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:37:49,219 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:37:49,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:51,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:53,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:37:53,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:37:53,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:37:53,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:37:53,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:55,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:57,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:37:58,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:37:59,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:00,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:38:00,647 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:38:00,647 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:38:00,649 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:38:01,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:01,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:38:03,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:04,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:38:05,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:07,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:08,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:38:08,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:38:08,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:38:09,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:38:09,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:11,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:13,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:14,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:38:15,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:17,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:19,234 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:38:19,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:20,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:38:21,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:23,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:38:23,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:38:23,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:38:23,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:25,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:25,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:38:27,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:29,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:31,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:38:31,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:33,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:35,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:36,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:38:37,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:38,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:38:38,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:38:38,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:38:39,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:41,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:41,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:38:43,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:45,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:38:45,455 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:38:45,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:38:45,457 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:38:45,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:45,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:38:47,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:38:47,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:49,223 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:38:49,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:51,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:52,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:38:53,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:38:53,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:38:53,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:38:53,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:55,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:57,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:38:57,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:38:59,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:01,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:03,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:39:03,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:05,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:07,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:08,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:39:08,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:39:08,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:39:08,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:39:09,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:11,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:13,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:13,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:39:15,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:17,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:19,226 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:39:19,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:39:19,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:21,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:23,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:39:23,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:39:23,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:39:23,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:24,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:39:25,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:27,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:29,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:30,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:39:30,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:39:30,169 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:39:30,169 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:39:30,170 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:39:30,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:39:31,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:33,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:35,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:39:35,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:37,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:38,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:39:38,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:39:38,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:39:39,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:40,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:39:41,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:43,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:45,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:46,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:39:47,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:49,228 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:39:49,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:51,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:39:51,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:53,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:39:53,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:39:53,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:39:53,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:55,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:57,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:39:57,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:39:59,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:01,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:02,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:40:03,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:05,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:07,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:07,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:40:08,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:40:08,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:40:08,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:40:09,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:11,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:13,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:40:13,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:14,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:40:14,990 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:40:14,990 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:40:14,992 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:40:15,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:15,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:40:17,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:18,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:40:19,242 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:40:19,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:21,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:23,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:40:23,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:40:23,377 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:40:23,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:23,931 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:40:25,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:27,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:29,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:40:29,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:31,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:33,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:34,734 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:40:35,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:37,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:38,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:40:38,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:40:38,407 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:40:39,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:40,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:40:41,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:43,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:45,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:40:45,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:47,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:49,233 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:40:49,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:50,853 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:40:51,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:53,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:40:53,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:40:53,388 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:40:53,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:55,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:56,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:40:57,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:59,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:40:59,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:40:59,844 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:40:59,844 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:40:59,847 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:41:00,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:41:01,604 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:41:01,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:03,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:05,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:06,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:41:07,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:08,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:41:08,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:41:08,388 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:41:09,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:11,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:12,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:41:13,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:15,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:17,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:41:17,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:19,235 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:41:19,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:21,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:23,123 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:41:23,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:41:23,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:41:23,388 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:41:23,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:25,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:27,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:28,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:41:29,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:31,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:33,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:33,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:41:35,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:37,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:38,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:41:38,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:41:38,388 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:41:39,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:41:39,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:41,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:43,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:44,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:41:44,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:41:44,691 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:41:44,691 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:41:44,693 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:41:44,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:41:45,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:47,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:49,237 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:41:49,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:50,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:41:51,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:53,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:41:53,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:41:53,388 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:41:53,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:55,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:41:55,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:57,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:41:58,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:00,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:42:00,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:02,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:04,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:06,098 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:42:06,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:08,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:42:08,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:42:08,388 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:42:08,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:10,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:11,461 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:42:12,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:14,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:16,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:42:16,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:18,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:19,249 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:42:20,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:22,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:42:22,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:23,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:42:23,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:42:23,388 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:42:24,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:26,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:27,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:42:28,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:29,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:42:29,425 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:42:29,425 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:42:29,427 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:42:29,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:42:30,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:31,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:32,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:32,984 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:42:34,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:36,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:38,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:42:38,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:42:38,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:42:38,443 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:42:38,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:40,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:42,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:43,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:42:44,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:46,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:48,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:49,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:42:49,241 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:42:50,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:52,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:53,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:42:53,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:42:53,423 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:42:54,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:42:54,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:56,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:58,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:42:59,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:43:00,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:02,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:04,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:05,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:43:06,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:08,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:43:08,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:43:08,424 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:43:08,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:10,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:43:10,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:12,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:14,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:43:14,220 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:43:14,221 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:43:14,222 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:43:14,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:14,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:43:15,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:15,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:43:16,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:18,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:19,244 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:43:20,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:21,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:43:22,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:23,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:43:23,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:43:23,424 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:43:24,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:26,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:43:26,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:28,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:30,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:32,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:43:32,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:34,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:36,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:37,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:43:38,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:43:38,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:43:38,424 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:43:38,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:40,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:42,809 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:43:42,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:44,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:46,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:48,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:43:48,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:49,246 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:43:50,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:52,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:53,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:43:53,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:43:53,424 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:43:53,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:43:55,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:57,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:59,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:43:59,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:43:59,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:43:59,033 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:43:59,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:43:59,035 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:44:00,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:44:01,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:02,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:03,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:04,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:44:05,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:07,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:08,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:44:08,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:44:08,424 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:44:09,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:09,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:44:11,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:13,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:15,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:15,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:44:17,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:19,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:19,258 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:44:20,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:44:21,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:23,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:23,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:44:23,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:44:23,424 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:44:25,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:25,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:44:27,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:29,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:31,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:31,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:44:33,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:35,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:36,588 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:44:37,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:38,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:44:38,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:44:38,473 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:44:39,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:41,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:41,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:44:43,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:43,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:44:43,828 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:44:43,829 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:44:43,830 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:44:44,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:44:45,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:46,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:47,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:47,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:44:49,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:49,250 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:44:51,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:52,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:44:53,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:53,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:44:53,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:44:53,453 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:44:55,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:57,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:44:58,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:44:59,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:01,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:03,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:03,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:45:05,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:07,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:08,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:45:08,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:45:08,453 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:45:08,603 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:45:09,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:11,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:13,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:14,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:45:15,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:17,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:19,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:19,252 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:45:19,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:45:21,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:23,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:23,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:45:23,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:45:23,453 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:45:25,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:45:25,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:27,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:28,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:45:28,669 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:45:28,669 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:45:28,671 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:45:29,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:29,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:45:30,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:45:31,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:33,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:35,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:35,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:45:37,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:38,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:45:38,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:45:38,453 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:45:39,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:41,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:45:41,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:43,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:45,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:46,620 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:45:47,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:49,255 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:45:49,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:51,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:52,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:45:53,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:45:53,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:53,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:45:53,453 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:45:55,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:57,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:45:57,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:45:59,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:01,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:02,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:46:03,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:05,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:07,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:08,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:46:08,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:46:08,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:46:08,453 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:46:09,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:11,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:13,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:13,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:46:13,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:46:13,551 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:46:13,551 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:46:13,552 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:46:14,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:46:15,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:17,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:18,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:46:19,268 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:46:19,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:21,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:23,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:46:23,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:23,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:46:23,453 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:46:24,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:46:25,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:27,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:29,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:29,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:46:31,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:33,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:35,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:46:35,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:37,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:38,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:46:38,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:46:38,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:46:39,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:40,521 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:46:41,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:43,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:45,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:45,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:46:47,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:49,259 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:46:49,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:51,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:46:51,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:53,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:46:53,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:46:53,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:46:53,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:55,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:56,629 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:46:57,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:46:58,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:46:58,459 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:46:58,459 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:46:58,461 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:46:58,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:46:59,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:01,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:02,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:47:03,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:05,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:07,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:47:07,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:08,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:47:08,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:47:08,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:47:09,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:11,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:12,805 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:47:13,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:15,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:17,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:18,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:47:19,261 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:47:19,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:21,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:23,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:47:23,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:47:23,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:47:23,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:23,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:47:25,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:27,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:28,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:47:29,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:31,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:33,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:34,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:47:35,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:37,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:38,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:47:38,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:47:38,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:47:39,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:39,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:47:41,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:43,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:47:43,211 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:47:43,211 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:47:43,213 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:47:43,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:43,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:47:44,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:47:45,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:47,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:49,263 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:47:49,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:50,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:47:51,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:53,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:47:53,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:47:53,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:47:53,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:55,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:55,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:47:57,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:47:59,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:00,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:48:01,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:03,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:05,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:06,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:48:07,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:08,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:48:08,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:48:08,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:48:09,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:11,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:11,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:48:13,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:15,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:16,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:48:17,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:19,266 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:48:19,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:21,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:22,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:48:23,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:48:23,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:48:23,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:48:23,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:25,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:27,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:27,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:48:27,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:48:27,642 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:48:27,642 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:48:27,644 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:48:28,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:48:29,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:31,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:32,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:48:33,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:35,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:37,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:38,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:48:38,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:48:38,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:48:38,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:48:39,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:41,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:43,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:43,606 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:48:45,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:47,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:48,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:48:49,268 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:48:49,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:51,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:53,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:48:53,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:48:53,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:48:53,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:54,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:48:55,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:57,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:59,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:48:59,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:49:01,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:03,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:05,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:49:05,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:07,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:08,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:49:08,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:49:08,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:49:09,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:10,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:49:11,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:12,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:49:12,166 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:49:12,166 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:49:12,168 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:49:12,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:49:13,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:15,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:15,686 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:49:17,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:19,270 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:49:19,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:20,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:49:21,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:23,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:49:23,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:49:23,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:49:23,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:25,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:26,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:49:27,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:29,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:31,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:49:31,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:33,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:35,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:36,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:49:37,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:38,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:49:38,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:49:38,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:49:39,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:41,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:42,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:49:43,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:45,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:47,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:49:47,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:49,274 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:49:49,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:51,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:52,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:49:53,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:49:53,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:49:53,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:49:53,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:55,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:56,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:49:56,541 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:49:56,542 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:49:56,544 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:49:56,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:49:57,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:49:58,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:49:59,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:01,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:03,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:50:03,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:05,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:07,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:08,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:50:08,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:50:08,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:50:08,622 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:50:09,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:11,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:13,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:14,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:50:15,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:17,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:19,276 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:50:19,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:50:19,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:21,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:23,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:50:23,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:50:23,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:50:23,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:24,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:50:25,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:27,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:29,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:30,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:50:31,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:33,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:35,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:50:35,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:37,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:38,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:50:38,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:50:38,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:50:39,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:40,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:50:40,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:50:40,979 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:50:40,980 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:50:40,981 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:50:41,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:41,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:50:43,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:45,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:46,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:50:47,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:49,279 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:50:49,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:51,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:50:51,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:53,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:50:53,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:50:53,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:50:53,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:55,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:56,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:50:57,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:50:59,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:01,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:02,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:51:03,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:05,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:07,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:51:07,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:08,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:51:08,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:51:08,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:51:09,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:11,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:12,835 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:51:13,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:15,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:17,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:18,165 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:51:19,281 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:51:19,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:21,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:23,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:51:23,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:51:23,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:51:23,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:51:23,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:25,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:51:25,317 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:51:25,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:51:25,321 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:51:25,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:25,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:51:27,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:28,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:51:29,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:31,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:33,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:34,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:51:35,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:37,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:38,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:51:38,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:51:38,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:51:39,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:51:39,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:41,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:43,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:44,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:51:45,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:47,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:49,283 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:51:49,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:50,107 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:51:51,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:53,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:51:53,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:51:53,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:51:53,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:55,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:51:55,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:57,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:51:59,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:00,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:52:01,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:03,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:05,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:06,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:52:07,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:08,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:52:08,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:52:08,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:52:09,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:52:09,697 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:52:09,697 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:52:09,699 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:52:09,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:09,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:52:11,449 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:52:11,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:13,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:15,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:16,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:52:17,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:19,285 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:52:19,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:21,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:22,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:52:23,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:52:23,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:52:23,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:52:23,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:25,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:27,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:52:27,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:29,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:31,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:32,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:52:33,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:35,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:37,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:38,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:52:38,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:52:38,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:52:38,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:52:39,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:41,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:43,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:52:43,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:45,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:47,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:48,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:52:49,298 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:52:49,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:51,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:53,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:52:53,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:52:53,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:52:53,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:54,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:52:54,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:52:54,059 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:52:54,060 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:52:54,062 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:52:54,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:52:55,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:57,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:52:59,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:52:59,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:01,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:03,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:04,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:53:05,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:07,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:08,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:53:08,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:53:08,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:53:09,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:10,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:53:11,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:13,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:15,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:53:15,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:17,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:19,290 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:53:19,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:20,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:53:21,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:23,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:53:23,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:53:23,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:53:23,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:25,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:26,014 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:53:27,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:29,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:31,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:53:31,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:33,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:35,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:36,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:53:37,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:38,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:53:38,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:53:38,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:53:38,495 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:53:38,586 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:53:38,587 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:53:38,588 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:53:38,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:53:39,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:41,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:42,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:53:43,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:45,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:47,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:53:47,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:49,292 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:53:49,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:51,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:52,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:53:53,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:53:53,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:53:53,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:53:53,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:55,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:57,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:53:57,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:53:59,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:01,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:03,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:54:03,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:05,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:07,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:08,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:54:08,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:54:08,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:54:08,598 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:54:09,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:11,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:13,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:54:13,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:15,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:17,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:19,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:54:19,294 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:54:19,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:21,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:22,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:54:22,877 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:54:22,877 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:54:22,879 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:54:22,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:54:23,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:54:23,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:54:23,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:54:23,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:24,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:54:25,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:27,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:29,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:54:29,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:31,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:33,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:35,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:54:35,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:37,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:38,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:54:38,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:54:38,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:54:39,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:40,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:54:41,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:43,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:45,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:54:45,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:47,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:49,308 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:54:49,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:51,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:54:51,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:53,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:54:53,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:54:53,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:54:53,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:55,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:54:56,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:54:57,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:00,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:01,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:55:02,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:04,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:06,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:07,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:55:07,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:55:07,242 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:55:07,242 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:55:07,244 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:55:08,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:08,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:55:08,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:55:08,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:55:08,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:55:10,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:12,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:12,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:55:14,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:16,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:17,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:55:18,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:19,299 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:55:20,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:22,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:23,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:55:23,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:55:23,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:55:23,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:55:24,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:26,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:28,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:28,558 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:55:29,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:31,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:33,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:33,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:55:35,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:37,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:38,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:55:38,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:55:38,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:55:39,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:55:39,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:41,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:43,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:44,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:55:45,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:47,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:49,301 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:55:49,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:49,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:55:51,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:51,598 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:55:51,600 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:55:51,601 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:55:51,603 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:55:52,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:55:53,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:55:53,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:53,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:55:53,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:55:55,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:55:55,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:57,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:55:59,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:00,450 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:56:01,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:03,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:05,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:05,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:56:07,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:08,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:56:08,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:56:08,458 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:56:09,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:11,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:56:11,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:13,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:15,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:16,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:56:17,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:19,304 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:56:19,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:21,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:21,815 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:56:23,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:56:23,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:23,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:56:23,458 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:56:25,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:27,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:56:27,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:29,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:31,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:32,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:56:33,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:35,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:36,098 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:56:36,100 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:56:36,101 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:56:36,103 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:56:36,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:56:37,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:37,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:56:38,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:56:38,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:56:38,458 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:56:39,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:41,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:43,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:56:43,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:45,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:47,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:48,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:56:49,316 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:56:49,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:51,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:53,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:56:53,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:53,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:56:53,458 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:56:53,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:56:55,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:57,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:56:59,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:56:59,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:01,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:03,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:04,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:57:05,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:07,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:08,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:57:08,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:57:08,458 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:57:09,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:09,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:57:11,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:13,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:15,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:57:15,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:17,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:19,308 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:57:19,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:20,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:57:20,459 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:57:20,461 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:57:20,461 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:57:20,463 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:57:21,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:21,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:57:23,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:57:23,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:23,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:57:23,458 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:57:25,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:26,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:57:27,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:29,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:31,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:31,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:57:33,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:36,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:57:39,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:57:39,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:57:39,776 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:57:42,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:57:43,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:45,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:47,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:48,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:57:49,311 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:57:49,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:51,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:53,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:54,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:57:54,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:57:54,754 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:57:54,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:57:55,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:57,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:57:59,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:00,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:58:01,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:03,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:05,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:05,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:58:07,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:09,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:09,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:58:09,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:58:09,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:58:11,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:11,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:58:13,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:15,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:16,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:58:16,529 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:58:16,529 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:58:16,531 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:58:17,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:17,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:58:17,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:58:19,313 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:58:19,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:21,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:23,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:58:23,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:24,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:58:24,754 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:58:24,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:58:25,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:27,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:29,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:58:29,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:31,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:33,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:34,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:58:35,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:37,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:39,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:39,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:58:39,754 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:58:39,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:58:39,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:58:41,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:43,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:45,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:58:45,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:47,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:49,327 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:58:49,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:50,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:58:51,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:53,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:54,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:58:54,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:58:54,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:58:55,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:55,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:58:57,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:58:59,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:01,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:59:01,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:03,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:04,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:59:04,331 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:59:04,331 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:59:04,333 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:59:04,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:59:05,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:07,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:59:07,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:09,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:09,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:59:09,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:59:09,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:59:11,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:12,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:59:13,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:15,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:17,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:18,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:59:19,317 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:59:19,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:21,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:23,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:24,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:59:24,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:59:24,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:59:24,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:59:25,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:27,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:29,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:29,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:59:31,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:33,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:35,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:35,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:59:37,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:39,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:39,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:59:39,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:59:39,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:59:41,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:59:41,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:43,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:45,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:46,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:59:47,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:49,320 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 15:59:49,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:51,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:51,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 15:59:51,631 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 15:59:51,632 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 15:59:51,633 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 15:59:52,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 15:59:52,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:59:53,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:54,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 15:59:54,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 15:59:54,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 15:59:55,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:57,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 15:59:58,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 15:59:59,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:01,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:03,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:03,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:00:05,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:07,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:09,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:00:09,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:09,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:00:09,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:00:09,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:00:11,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:13,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:15,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:00:15,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:17,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:19,322 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:00:19,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:20,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:00:21,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:23,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:24,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:00:24,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:00:24,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:00:25,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:26,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:00:27,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:29,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:31,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:32,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:00:33,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:35,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:37,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:37,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:00:38,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:00:38,654 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:00:38,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:00:38,655 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:00:39,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:39,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:00:39,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:00:39,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:00:39,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:00:41,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:43,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:00:43,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:45,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:47,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:48,961 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:00:49,334 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:00:49,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:51,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:53,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:54,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:00:54,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:00:54,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:00:54,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:00:55,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:57,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:00:59,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:00,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:01:01,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:03,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:05,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:05,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:01:07,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:09,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:09,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:01:09,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:01:09,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:01:11,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:01:11,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:13,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:15,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:16,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:01:17,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:19,327 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:01:19,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:21,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:22,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:01:23,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:24,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:01:24,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:01:24,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:01:25,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:01:25,207 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:01:25,207 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:01:25,209 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:01:25,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:25,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:01:27,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:28,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:01:29,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:31,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:33,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:01:33,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:35,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:37,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:39,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:01:39,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:39,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:01:39,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:01:39,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:01:41,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:43,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:44,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:01:45,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:47,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:49,329 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:01:49,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:50,427 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:01:51,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:53,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:54,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:01:54,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:01:54,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:01:55,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:56,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:01:57,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:01:59,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:01,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:02:01,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:03,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:05,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:07,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:02:07,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:09,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:09,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:02:09,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:02:09,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:02:11,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:11,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:02:11,788 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:02:11,789 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:02:11,792 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:02:12,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:02:12,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:02:13,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:15,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:17,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:18,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:02:19,330 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:02:19,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:21,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:23,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:23,898 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:02:24,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:02:24,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:02:24,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:02:25,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:27,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:29,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:02:29,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:31,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:33,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:35,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:02:35,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:37,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:39,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:02:39,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:02:39,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:02:39,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:40,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:02:41,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:43,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:45,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:46,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:02:47,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:49,332 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:02:49,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:51,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:02:51,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:53,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:54,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:02:54,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:02:54,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:02:55,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:57,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:02:57,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:02:58,065 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:02:58,067 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:02:58,067 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:02:58,069 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:02:58,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:02:59,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:01,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:02,721 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:03:03,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:05,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:07,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:08,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:03:09,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:03:09,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:03:09,757 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:03:09,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:11,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:13,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:13,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:03:15,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:17,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:19,335 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:03:19,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:03:19,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:21,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:23,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:24,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:03:24,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:03:24,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:03:24,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:03:25,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:27,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:29,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:30,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:03:31,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:33,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:35,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:36,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:03:37,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:39,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:03:39,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:03:39,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:03:39,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:41,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:03:41,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:43,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:44,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:03:44,290 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:03:44,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:03:44,292 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:03:44,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:03:45,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:47,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:03:47,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:49,337 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:03:49,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:51,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:52,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:03:53,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:54,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:03:54,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:03:54,757 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:03:55,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:57,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:03:58,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:03:59,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:01,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:03,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:04:03,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:05,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:07,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:09,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:04:09,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:04:09,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:04:09,757 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:04:09,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:11,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:13,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:14,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:04:15,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:17,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:19,339 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:04:19,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:20,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:04:21,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:23,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:24,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:04:24,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:04:24,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:04:25,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:25,921 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:04:27,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:29,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:30,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:04:30,248 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:04:30,248 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:04:30,250 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:04:30,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:04:31,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:04:31,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:33,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:35,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:36,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:04:37,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:39,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:04:39,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:04:39,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:04:39,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:41,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:42,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:04:43,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:45,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:47,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:04:47,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:49,341 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:04:49,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:51,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:53,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:04:53,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:54,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:04:54,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:04:54,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:04:55,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:57,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:04:58,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:04:59,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:01,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:03,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:04,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:05:05,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:07,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:09,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:05:09,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:05:09,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:05:09,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:05:09,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:11,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:13,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:15,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:05:15,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:16,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:05:16,326 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:05:16,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:05:16,328 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:05:16,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:05:17,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:19,343 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:05:19,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:20,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:05:21,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:23,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:24,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:05:24,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:05:24,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:05:25,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:26,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:05:27,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:29,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:31,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:32,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:05:33,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:35,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:37,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:05:37,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:39,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:05:39,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:05:39,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:05:39,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:41,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:43,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:05:43,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:45,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:47,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:48,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:05:49,346 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:05:49,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:52,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:54,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:54,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:05:54,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:05:54,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:05:54,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:05:56,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:58,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:05:59,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:06:00,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:02,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:02,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:06:02,348 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:06:02,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:06:02,350 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:06:03,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:06:04,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:05,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:06:06,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:08,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:09,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:06:09,779 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:06:09,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:06:10,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:10,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:06:12,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:14,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:16,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:16,030 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:06:18,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:19,348 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:06:20,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:21,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:06:22,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:24,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:24,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:06:24,779 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:06:24,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:06:26,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:27,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:06:28,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:30,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:32,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:32,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:06:34,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:36,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:37,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:06:38,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:39,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:06:39,779 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:06:39,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:06:40,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:42,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:43,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:06:44,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:46,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:48,047 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:06:48,049 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:06:48,050 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:06:48,051 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:06:48,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:48,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:06:49,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:06:49,351 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:06:50,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:52,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:54,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:54,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:06:54,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:06:54,779 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:06:54,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:06:56,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:06:58,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:00,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:07:00,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:02,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:04,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:05,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:07:06,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:08,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:09,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:07:09,780 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:07:09,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:07:10,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:10,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:07:12,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:14,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:16,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:16,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:07:18,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:19,364 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:07:20,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:21,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:07:22,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:24,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:24,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:07:24,779 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:07:24,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:07:26,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:27,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:07:28,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:30,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:32,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:32,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:07:33,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:07:33,774 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:07:33,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:07:33,775 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:07:34,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:34,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:07:36,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:38,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:38,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:07:39,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:39,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:07:39,779 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:07:39,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:07:41,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:43,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:43,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:07:45,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:47,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:49,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:07:49,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:49,355 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:07:51,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:53,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:54,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:07:54,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:07:54,780 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:07:54,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:07:55,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:57,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:07:59,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:00,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:08:01,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:03,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:05,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:05,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:08:07,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:09,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:09,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:08:09,780 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:08:09,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:08:11,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:08:11,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:13,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:15,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:16,605 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:08:17,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:19,263 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:08:19,265 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:08:19,265 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:08:19,268 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:08:19,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:19,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:08:19,357 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:08:21,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:22,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:08:23,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:24,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:08:24,784 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:08:24,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:08:25,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:27,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:27,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:08:29,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:31,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:32,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:08:33,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:35,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:37,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:38,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:08:39,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:39,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:08:39,784 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:08:39,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:08:41,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:43,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:43,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:08:45,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:47,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:49,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:08:49,359 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:08:49,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:51,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:53,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:54,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:08:54,784 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:08:54,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:08:54,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:08:55,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:57,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:08:59,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:00,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:09:01,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:03,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:04,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:09:04,726 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:09:04,726 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:09:04,728 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:09:05,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:05,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:09:05,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:09:07,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:09,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:09,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:09:09,784 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:09:09,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:09:11,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:09:11,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:13,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:15,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:16,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:09:17,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:19,372 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:09:19,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:21,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:22,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:09:23,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:24,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:09:24,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:09:24,849 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:09:25,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:27,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:27,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:09:29,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:31,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:33,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:09:33,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:35,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:37,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:38,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:09:39,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:39,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:09:39,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:09:39,829 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:09:41,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:43,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:43,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:09:45,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:47,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:49,364 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:09:49,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:09:49,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:50,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:09:50,219 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:09:50,220 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:09:50,221 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:09:50,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:09:51,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:53,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:54,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:09:54,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:09:54,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:09:54,829 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:09:55,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:57,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:09:59,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:00,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:10:01,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:03,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:05,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:05,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:10:07,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:09,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:09,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:10:09,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:10:09,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:10:11,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:10:11,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:13,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:15,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:16,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:10:17,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:19,366 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:10:19,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:21,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:22,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:10:23,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:24,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:10:24,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:10:24,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:10:25,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:27,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:27,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:10:29,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:31,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:33,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:10:33,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:35,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:35,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:10:35,753 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:10:35,754 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:10:35,755 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:10:36,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:10:37,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:38,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:10:39,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:39,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:10:39,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:10:39,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:10:41,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:43,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:43,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:10:45,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:47,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:49,369 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:10:49,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:10:49,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:51,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:53,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:54,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:10:54,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:10:54,833 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:10:54,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:10:55,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:57,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:10:59,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:00,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:11:01,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:03,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:05,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:05,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:11:07,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:09,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:09,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:11:09,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:11:09,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:11:11,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:11:11,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:13,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:15,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:16,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:11:17,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:19,383 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:11:19,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:21,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:11:21,239 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:11:21,239 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:11:21,241 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:11:21,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:21,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:11:22,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:11:23,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:24,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:11:24,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:11:24,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:11:25,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:27,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:27,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:11:29,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:31,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:33,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:11:33,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:35,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:37,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:38,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:11:39,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:39,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:11:39,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:11:39,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:11:41,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:43,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:44,117 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:11:45,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:47,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:49,373 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:11:49,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:11:49,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:51,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:53,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:54,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:11:54,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:11:54,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:11:54,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:11:55,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:57,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:11:59,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:00,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:12:01,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:03,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:05,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:06,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:12:06,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:12:06,913 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:12:06,914 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:12:06,916 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:12:07,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:07,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:12:09,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:09,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:12:09,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:12:09,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:12:11,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:12:11,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:13,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:15,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:17,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:12:17,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:19,375 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:12:19,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:21,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:22,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:12:23,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:24,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:12:24,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:12:24,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:12:25,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:27,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:28,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:12:29,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:31,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:33,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:12:33,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:35,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:37,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:39,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:12:39,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:39,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:12:39,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:12:39,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:12:41,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:43,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:44,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:12:45,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:47,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:49,377 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:12:49,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:50,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:12:51,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:52,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:12:52,800 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:12:52,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:12:52,804 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:12:53,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:53,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:12:54,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:12:54,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:12:54,831 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:12:55,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:55,985 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:12:57,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:12:59,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:01,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:13:01,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:03,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:05,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:06,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:13:07,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:09,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:09,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:13:09,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:13:09,831 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:13:11,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:12,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:13:13,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:15,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:17,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:13:17,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:19,392 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:13:19,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:21,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:23,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:13:23,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:24,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:13:24,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:13:24,831 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:13:25,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:27,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:28,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:13:29,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:31,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:33,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:34,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:13:35,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:37,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:38,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:13:38,604 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:13:38,604 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:13:38,606 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:13:38,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:13:39,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:13:39,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:39,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:13:39,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:13:39,831 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:13:41,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:43,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:45,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:13:45,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:47,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:49,382 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:13:49,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:50,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:13:51,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:53,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:54,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:13:54,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:13:54,831 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:13:55,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:56,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:13:57,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:13:59,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:01,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:14:01,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:03,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:05,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:07,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:14:07,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:09,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:09,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:14:09,857 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:14:09,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:14:11,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:12,582 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:14:13,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:15,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:17,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:18,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:14:19,384 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:14:19,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:21,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:23,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:14:23,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:24,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:14:24,423 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:14:24,423 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:14:24,425 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:14:24,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:14:24,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:14:24,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:14:24,844 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:14:25,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:27,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:29,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:14:29,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:31,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:33,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:34,517 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:14:35,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:37,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:39,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:39,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:14:39,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:14:39,845 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:14:39,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:14:41,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:43,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:45,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:14:45,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:47,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:49,385 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:14:49,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:50,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:14:51,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:53,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:54,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:14:54,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:14:54,845 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:14:55,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:56,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:14:57,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:14:59,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:01,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:01,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:15:03,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:05,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:07,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:15:07,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:09,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:09,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:15:09,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:15:09,845 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:15:09,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:15:09,943 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:15:09,944 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:15:09,945 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:15:10,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:15:11,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:12,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:15:13,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:15,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:17,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:18,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:15:19,388 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:15:19,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:21,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:23,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:15:23,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:24,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:15:24,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:15:24,845 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:15:25,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:27,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:29,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:15:29,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:31,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:33,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:34,669 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:15:35,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:37,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:39,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:39,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:15:39,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:15:39,845 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:15:39,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:15:41,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:43,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:45,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:15:45,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:47,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:49,390 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:15:49,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:51,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:15:51,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:53,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:54,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:15:54,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:15:54,845 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:15:55,586 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:15:55,587 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:15:55,587 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:15:55,589 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:15:55,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:55,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:15:56,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:15:57,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:15:59,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:01,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:02,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:16:03,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:05,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:07,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:16:07,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:09,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:16:09,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:16:09,845 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:16:09,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:11,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:12,921 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:16:13,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:15,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:17,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:18,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:16:19,393 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:16:19,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:21,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:23,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:16:23,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:24,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:16:24,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:16:24,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:16:25,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:27,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:29,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:16:29,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:31,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:33,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:34,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:16:35,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:37,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:39,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:16:39,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:16:39,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:16:39,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:40,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:16:41,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:16:41,127 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:16:41,127 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:16:41,129 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:16:41,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:41,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:16:42,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:16:42,713 DEBUG SenderThread:156911 [sender.py:send():379] send: metric -2024-04-25 16:16:42,714 DEBUG SenderThread:156911 [sender.py:send():379] send: metric -2024-04-25 16:16:42,714 DEBUG SenderThread:156911 [sender.py:send():379] send: metric -2024-04-25 16:16:42,715 DEBUG SenderThread:156911 [sender.py:send():379] send: metric -2024-04-25 16:16:42,715 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:16:42,715 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:16:42,718 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:16:42,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:16:43,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:45,727 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:16:47,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:49,395 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:16:51,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:16:54,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:16:54,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:16:54,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:16:56,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:16:57,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:16:59,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:01,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:01,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:17:02,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/config.yaml -2024-04-25 16:17:03,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:05,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:07,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:17:07,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:09,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:17:09,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:17:09,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:17:09,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:11,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:12,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:17:13,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:15,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:17,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:18,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:17:19,418 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:17:19,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:21,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:23,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:17:24,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:24,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:17:24,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:17:24,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:17:26,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:28,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:29,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:17:30,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:32,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:34,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:34,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:17:36,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:38,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:39,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:17:39,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:17:39,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:17:39,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:17:40,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:42,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:17:42,104 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:17:42,104 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:17:42,106 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:17:42,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:42,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:17:44,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:45,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:17:46,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:48,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:49,399 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:17:50,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:51,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:17:52,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:54,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:54,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:17:54,912 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:17:54,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:17:56,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:17:56,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:17:58,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:00,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:02,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:18:02,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:04,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:06,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:07,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:18:08,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:09,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:18:09,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:18:09,900 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:18:10,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:12,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:13,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:18:14,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:16,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:18,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:18,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:18:19,401 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:18:20,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:22,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:23,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:18:24,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:24,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:18:24,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:18:24,900 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:18:26,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:27,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:18:27,655 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:18:27,655 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:18:27,657 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:18:28,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:28,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:18:29,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:18:30,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:32,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:34,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:34,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:18:36,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:38,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:39,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:18:39,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:18:39,900 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:18:40,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:18:40,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:42,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:44,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:45,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:18:46,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:48,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:49,404 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:18:50,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:51,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:18:52,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:54,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:54,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:18:54,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:18:54,900 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:18:56,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:18:56,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:18:58,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:00,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:02,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:19:02,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:04,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:06,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:07,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:19:08,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:09,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:19:09,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:19:09,900 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:19:10,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:12,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:13,111 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:19:13,143 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:19:13,145 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:19:13,145 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:19:13,147 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:19:13,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:19:14,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:16,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:18,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:18,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:19:19,418 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:19:20,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:22,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:24,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:19:24,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:24,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:19:24,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:19:24,900 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:19:26,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:28,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:29,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:19:30,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:32,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:34,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:35,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:19:36,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:38,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:39,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:19:39,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:19:39,901 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:19:40,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:19:40,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:42,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:44,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:45,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:19:46,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:48,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:49,408 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:19:50,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:50,522 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:19:52,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:54,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:54,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:19:54,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:19:54,901 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:19:56,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:19:56,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:58,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:19:58,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:19:58,628 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:19:58,629 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:19:58,630 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:19:59,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:20:00,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:01,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:20:02,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:04,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:06,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:06,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:20:08,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:09,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:20:09,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:20:09,901 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:20:10,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:12,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:20:12,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:14,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:16,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:17,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:20:18,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:19,411 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:20:20,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:22,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:23,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:20:24,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:24,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:20:24,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:20:24,901 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:20:26,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:28,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:28,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:20:30,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:32,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:34,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:20:34,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:36,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:38,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:39,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:20:39,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:20:39,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:20:39,901 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:20:40,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:42,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:44,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:20:44,003 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:20:44,003 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:20:44,005 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:20:44,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:44,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:20:45,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:20:46,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:48,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:49,412 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:20:50,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:50,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:20:52,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:54,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:54,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:20:54,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:20:54,972 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:20:56,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:20:56,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:20:58,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:00,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:01,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:21:02,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:04,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:06,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:06,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:21:08,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:09,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:21:09,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:21:09,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:21:10,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:12,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:21:12,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:14,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:16,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:17,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:21:18,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:19,415 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:21:20,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:22,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:23,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:21:24,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:24,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:21:24,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:21:24,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:21:26,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:28,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:28,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:21:29,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:21:29,406 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:21:29,407 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:21:29,408 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:21:30,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:30,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:21:32,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:33,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:21:34,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:36,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:38,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:39,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:21:39,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:21:39,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:21:39,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:21:40,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:42,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:44,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:44,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:21:46,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:48,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:49,417 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:21:50,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:21:50,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:52,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:54,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:54,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:21:54,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:21:54,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:21:56,056 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:21:56,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:21:58,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:00,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:01,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:22:02,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:04,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:06,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:06,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:22:08,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:09,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:22:09,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:22:09,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:22:10,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:12,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:22:12,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:14,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:14,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:22:14,858 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:22:14,859 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:22:14,860 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:22:15,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:15,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:22:17,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:22:18,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:19,420 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:22:20,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:22,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:23,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:22:24,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:24,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:22:24,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:22:24,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:22:26,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:28,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:28,582 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:22:30,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:32,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:34,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:22:34,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:36,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:38,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:39,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:22:39,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:22:39,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:22:39,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:22:40,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:42,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:44,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:44,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:22:46,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:48,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:49,422 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:22:50,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:22:50,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:52,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:54,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:54,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:22:54,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:22:54,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:22:56,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:22:56,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:22:58,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:00,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:23:00,350 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:23:00,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:23:00,353 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:23:00,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:00,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:23:01,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:23:01,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:04,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:06,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:06,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:23:08,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:09,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:23:09,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:23:09,965 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:23:10,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:12,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:23:12,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:14,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:16,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:17,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:23:18,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:19,424 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:23:20,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:22,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:23,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:23:24,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:24,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:23:24,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:23:24,965 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:23:26,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:28,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:28,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:23:30,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:32,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:34,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:23:34,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:36,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:38,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:39,501 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:23:39,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:23:39,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:23:39,965 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:23:40,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:42,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:44,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:44,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:23:45,823 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:23:45,825 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:23:45,825 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:23:45,827 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:23:46,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:46,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:23:47,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:49,426 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:23:50,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:23:50,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:52,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:54,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:54,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:23:55,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:23:55,024 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:23:56,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:23:56,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:23:58,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:00,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:01,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:24:02,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:03,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:05,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:06,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:24:07,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:09,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:09,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:24:10,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:24:10,018 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:24:11,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:12,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:24:13,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:15,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:17,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:17,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:24:19,428 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:24:19,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:21,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:23,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:24:23,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:24,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:24:25,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:24:25,018 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:24:25,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:27,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:28,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:24:29,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:31,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:24:31,250 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:24:31,251 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:24:31,252 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:24:31,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:31,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:24:33,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:34,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:24:35,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:37,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:39,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:24:39,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:39,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:24:40,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:24:40,018 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:24:41,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:43,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:44,977 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:24:45,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:47,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:49,431 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:24:49,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:50,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:24:51,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:53,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:54,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:24:55,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:24:55,091 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:24:55,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:56,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:24:57,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:24:59,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:01,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:25:01,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:03,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:05,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:06,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:25:07,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:09,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:09,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:25:10,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:25:10,077 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:25:11,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:12,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:25:13,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:15,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:16,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:25:16,785 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:25:16,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:25:16,787 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:25:17,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:17,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:25:17,788 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:25:19,433 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:25:19,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:21,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:23,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:25:23,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:24,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:25:25,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:25:25,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:25:25,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:27,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:28,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:25:29,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:31,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:33,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:34,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:25:35,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:37,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:39,538 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:25:39,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:39,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:25:40,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:25:40,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:25:41,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:43,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:44,991 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:25:45,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:47,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:49,435 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:25:49,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:50,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:25:51,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:53,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:54,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:25:55,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:25:55,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:25:55,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:56,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:25:57,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:25:59,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:01,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:26:01,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:02,138 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:26:02,140 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:26:02,141 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:26:02,142 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:26:02,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:26:03,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:05,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:06,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:26:07,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:09,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:09,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:26:10,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:26:10,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:26:11,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:12,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:26:13,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:15,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:17,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:26:17,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:19,437 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:26:19,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:21,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:23,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:26:23,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:24,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:26:25,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:26:25,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:26:25,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:27,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:28,526 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:26:29,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:31,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:33,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:34,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:26:35,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:37,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:39,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:26:39,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:39,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:26:40,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:26:40,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:26:41,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:43,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:44,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:26:45,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:47,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:26:47,564 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:26:47,564 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:26:47,566 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:26:47,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:47,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:26:49,439 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:26:49,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:50,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:26:51,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:53,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:54,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:26:55,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:26:55,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:26:55,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:56,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:26:57,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:26:59,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:01,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:27:01,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:03,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:05,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:06,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:27:07,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:09,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:09,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:27:10,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:27:10,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:27:11,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:12,185 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:27:13,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:15,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:17,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:27:17,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:19,441 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:27:19,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:21,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:23,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:27:23,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:24,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:27:25,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:27:25,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:27:25,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:27,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:28,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:27:29,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:31,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:32,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:27:32,969 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:27:32,970 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:27:32,972 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:27:33,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:27:34,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:34,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:27:36,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:38,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:39,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:27:39,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:27:40,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:40,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:27:40,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:27:42,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:44,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:44,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:27:46,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:48,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:49,455 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:27:50,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:50,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:27:52,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:54,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:54,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:27:55,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:27:55,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:27:56,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:27:56,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:27:58,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:00,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:01,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:28:02,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:04,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:06,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:07,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:28:08,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:09,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:28:10,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:28:10,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:28:10,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:12,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:12,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:28:14,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:16,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:18,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:18,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:28:18,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:28:18,366 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:28:18,366 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:28:18,369 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:28:19,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:28:19,445 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:28:20,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:22,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:23,809 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:28:24,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:24,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:28:25,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:28:25,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:28:26,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:28,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:29,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:28:30,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:32,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:34,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:34,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:28:36,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:38,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:39,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:28:40,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:28:40,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:28:40,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:28:40,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:42,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:44,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:45,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:28:46,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:48,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:49,448 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:28:50,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:51,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:28:52,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:54,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:54,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:28:55,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:28:55,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:28:56,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:28:56,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:28:58,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:00,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:01,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:29:02,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:03,788 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:29:03,790 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:29:03,790 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:29:03,792 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:29:04,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:04,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:29:06,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:07,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:29:08,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:09,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:29:10,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:29:10,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:29:10,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:12,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:12,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:29:14,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:16,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:18,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:18,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:29:19,450 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:29:20,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:22,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:23,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:29:24,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:24,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:29:25,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:29:25,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:29:26,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:28,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:29,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:29:30,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:32,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:34,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:34,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:29:36,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:38,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:39,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:29:40,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:29:40,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:29:40,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:29:40,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:42,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:44,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:45,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:29:46,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:48,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:49,262 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:29:49,264 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:29:49,264 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:29:49,266 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:29:49,521 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:29:50,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:50,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:29:51,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:29:52,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:54,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:54,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:29:55,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:29:55,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:29:56,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:29:56,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:29:58,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:00,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:01,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:30:02,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:04,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:06,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:07,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:30:08,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:09,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:30:10,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:30:10,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:30:10,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:12,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:12,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:30:14,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:16,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:18,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:18,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:30:19,512 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:30:20,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:22,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:23,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:30:24,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:24,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:30:25,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:30:25,080 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:30:26,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:28,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:29,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:30:30,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:32,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:34,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:34,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:30:34,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:30:34,582 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:30:34,582 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:30:34,584 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:30:35,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:30:36,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:38,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:39,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:30:39,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:30:40,098 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:30:40,098 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:30:40,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:42,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:44,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:45,432 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:30:46,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:48,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:49,514 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:30:50,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:50,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:30:52,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:54,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:54,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:30:55,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:30:55,092 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:30:56,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:30:56,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:30:58,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:00,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:01,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:31:02,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:04,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:06,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:07,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:31:08,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:09,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:31:10,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:31:10,092 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:31:10,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:12,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:12,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:31:14,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:16,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:18,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:31:18,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:19,517 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:31:19,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:31:19,896 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:31:19,897 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:31:19,899 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:31:20,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:20,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:31:22,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:23,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:31:24,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:24,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:31:25,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:31:25,092 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:31:26,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:28,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:28,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:31:30,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:32,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:34,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:34,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:31:36,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:38,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:39,788 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:31:39,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:31:40,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:31:40,092 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:31:40,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:42,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:44,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:45,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:31:46,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:48,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:49,530 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:31:50,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:50,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:31:52,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:54,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:54,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:31:55,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:31:55,092 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:31:56,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:31:56,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:31:58,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:00,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:01,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:32:02,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:04,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:05,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:32:05,189 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:32:05,190 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:32:05,192 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:32:05,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:32:06,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:06,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:32:08,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:09,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:32:10,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:32:10,092 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:32:10,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:12,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:12,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:32:14,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:16,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:17,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:32:18,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:19,521 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:32:20,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:22,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:23,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:32:24,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:24,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:32:25,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:32:25,115 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:32:26,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:28,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:28,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:32:30,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:32,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:34,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:32:34,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:36,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:38,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:39,526 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:32:39,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:32:40,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:32:40,095 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:32:40,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:42,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:44,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:44,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:32:46,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:48,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:49,523 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:32:50,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:50,431 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:32:50,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:32:50,465 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:32:50,465 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:32:50,467 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:32:51,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:32:52,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:54,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:54,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:32:55,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:32:55,095 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:32:55,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:32:56,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:32:58,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:00,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:01,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:33:02,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:04,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:06,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:06,726 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:33:08,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:09,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:33:10,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:33:10,095 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:33:10,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:12,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:33:12,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:14,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:16,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:17,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:33:18,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:19,526 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:33:20,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:22,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:22,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:33:24,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:24,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:33:25,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:33:25,096 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:33:26,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:28,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:28,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:33:30,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:32,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:33,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:33:34,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:35,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:33:35,755 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:33:35,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:33:35,759 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:33:36,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:36,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:33:38,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:39,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:33:39,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:33:40,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:33:40,096 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:33:40,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:42,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:44,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:44,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:33:46,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:48,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:49,541 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:33:50,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:33:50,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:52,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:54,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:54,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:33:55,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:33:55,096 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:33:55,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:33:56,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:33:58,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:00,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:34:00,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:02,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:04,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:05,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:34:06,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:08,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:09,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:34:10,168 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:34:10,168 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:34:10,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:11,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:34:12,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:14,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:16,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:16,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:34:18,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:19,531 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:34:20,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:20,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:34:20,972 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:34:20,973 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:34:20,975 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:34:21,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:34:21,975 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:34:22,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:24,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:24,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:34:25,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:34:25,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:34:26,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:27,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:34:28,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:30,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:32,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:32,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:34:34,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:36,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:38,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:34:38,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:39,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:34:40,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:34:40,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:34:40,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:42,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:43,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:34:44,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:46,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:48,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:49,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:34:49,533 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:34:50,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:52,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:54,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:54,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:34:54,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:34:55,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:34:55,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:34:56,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:34:58,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:00,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:35:00,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:02,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:04,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:05,519 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:35:06,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:35:06,368 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:35:06,369 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:35:06,371 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:35:06,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:06,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:35:08,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:09,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:35:10,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:35:10,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:35:10,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:11,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:35:12,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:14,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:16,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:35:16,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:18,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:19,535 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:35:20,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:21,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:35:22,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:24,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:24,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:35:25,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:35:25,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:35:26,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:27,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:35:28,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:30,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:32,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:32,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:35:34,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:36,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:38,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:35:38,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:39,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:35:40,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:35:40,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:35:40,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:42,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:43,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:35:44,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:46,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:48,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:49,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:35:49,548 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:35:50,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:51,690 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:35:51,692 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:35:51,692 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:35:51,694 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:35:52,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:52,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:35:53,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:54,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:35:54,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:35:55,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:35:55,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:35:56,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:58,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:35:59,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:36:00,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:02,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:04,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:05,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:36:06,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:08,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:09,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:36:10,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:36:10,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:36:10,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:11,233 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:36:12,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:14,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:16,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:16,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:36:18,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:19,540 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:36:20,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:22,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:36:22,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:24,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:24,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:36:25,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:36:25,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:36:26,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:27,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:36:28,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:30,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:32,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:33,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:36:34,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:36,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:37,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:36:37,007 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:36:37,007 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:36:37,009 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:36:37,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:37,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:36:38,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:36:39,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:36:40,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:36:40,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:36:40,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:42,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:44,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:36:44,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:46,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:48,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:49,545 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:36:49,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:36:50,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:52,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:54,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:54,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:36:55,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:36:55,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:36:55,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:36:56,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:36:58,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:00,432 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:37:00,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:02,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:04,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:05,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:37:06,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:08,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:09,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:37:10,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:37:10,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:37:10,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:11,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:37:12,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:14,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:16,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:16,721 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:37:18,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:19,547 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:37:20,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:22,138 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:37:22,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:37:22,172 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:37:22,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:37:22,174 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:37:22,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:22,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:37:23,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:24,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:37:25,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:37:25,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:37:26,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:27,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:37:28,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:30,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:32,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:32,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:37:34,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:35,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:37,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:38,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:37:39,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:39,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:37:40,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:37:40,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:37:41,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:43,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:37:43,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:45,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:47,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:49,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:37:49,549 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:37:49,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:51,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:53,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:54,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:37:54,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:37:55,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:37:55,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:37:55,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:57,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:37:59,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:00,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:38:01,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:03,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:05,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:38:05,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:07,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:38:07,320 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:38:07,320 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:38:07,321 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:38:07,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:07,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:38:09,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:09,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:38:10,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:38:10,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:38:10,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:38:11,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:13,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:15,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:16,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:38:17,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:19,551 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:38:19,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:21,725 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:38:21,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:23,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:24,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:38:25,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:38:25,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:38:25,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:27,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:38:27,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:29,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:31,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:32,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:38:33,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:35,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:37,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:38,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:38:39,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:39,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:38:40,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:38:40,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:38:41,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:43,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:38:43,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:45,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:47,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:48,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:38:49,553 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:38:49,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:51,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:52,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:38:52,589 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:38:52,589 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:38:52,591 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:38:52,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:38:53,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:54,369 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:38:54,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:38:55,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:38:55,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:38:55,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:57,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:38:59,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:38:59,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:01,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:03,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:05,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:39:05,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:07,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:09,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:39:09,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:10,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:39:10,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:39:10,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:39:11,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:13,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:15,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:16,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:39:17,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:19,556 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:39:19,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:21,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:39:21,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:23,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:24,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:39:25,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:39:25,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:39:25,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:26,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:39:27,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:29,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:31,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:32,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:39:33,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:35,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:37,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:39:37,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:39:37,770 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:39:37,771 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:39:37,772 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:39:38,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:38,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:39:39,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:39:40,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:40,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:39:40,151 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:39:42,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:43,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:39:44,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:46,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:48,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:48,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:39:49,557 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:39:50,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:52,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:54,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:54,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:39:54,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:39:55,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:39:55,151 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:39:56,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:58,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:39:59,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:40:00,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:02,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:04,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:04,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:40:06,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:08,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:09,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:40:10,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:10,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:40:10,151 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:40:10,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:40:12,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:14,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:15,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:40:16,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:18,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:19,571 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:40:20,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:21,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:40:22,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:23,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:40:23,105 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:40:23,105 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:40:23,107 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:40:24,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:24,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:40:24,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:40:25,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:40:25,151 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:40:26,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:26,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:40:28,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:30,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:32,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:32,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:40:34,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:36,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:37,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:40:38,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:39,977 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:40:40,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:40,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:40:40,151 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:40:42,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:42,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:40:44,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:46,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:48,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:48,355 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:40:49,562 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:40:50,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:52,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:53,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:40:54,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:54,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:40:55,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:40:55,151 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:40:56,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:58,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:40:59,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:41:00,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:02,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:04,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:04,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:41:06,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:08,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:08,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:41:08,286 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:41:08,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:41:08,288 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:41:09,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:41:09,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:41:10,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:41:10,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:10,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:41:10,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:41:12,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:14,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:15,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:41:16,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:18,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:19,564 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:41:20,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:20,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:41:22,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:24,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:24,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:41:25,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:41:25,162 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:41:26,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:26,231 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:41:28,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:30,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:31,690 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:41:32,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:34,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:36,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:37,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:41:38,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:39,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:41:40,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:40,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:41:40,162 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:41:42,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:42,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:41:44,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:46,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:47,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:41:48,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:49,567 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:41:50,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:52,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:53,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:41:53,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:41:53,439 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:41:53,439 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:41:53,441 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:41:54,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:54,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:41:54,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:41:55,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:41:55,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:41:56,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:58,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:41:58,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:42:00,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:02,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:04,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:04,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:42:06,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:08,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:09,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:42:09,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:42:10,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:10,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:42:10,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:42:12,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:14,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:15,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:42:16,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:18,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:19,580 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:42:20,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:20,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:42:22,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:24,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:24,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:42:25,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:42:25,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:42:25,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:42:26,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:28,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:30,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:31,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:42:32,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:34,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:36,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:36,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:42:38,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:38,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:42:38,544 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:42:38,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:42:38,545 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:42:39,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:42:39,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:42:40,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:42:40,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:42:40,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:42,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:42:42,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:44,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:46,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:47,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:42:48,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:49,571 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:42:50,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:52,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:52,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:42:54,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:54,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:42:55,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:42:55,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:42:56,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:58,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:42:58,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:43:00,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:02,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:03,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:43:04,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:06,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:08,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:09,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:43:09,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:43:10,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:43:10,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:43:10,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:12,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:14,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:14,537 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:43:16,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:18,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:19,573 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:43:19,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:43:20,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:22,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:23,604 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:43:23,606 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:43:23,606 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:43:23,608 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:43:24,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:24,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:43:24,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:43:24,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:43:25,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:43:25,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:43:26,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:28,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:30,015 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:43:30,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:32,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:34,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:35,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:43:36,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:38,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:39,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:43:40,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:43:40,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:43:40,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:41,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:43:42,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:44,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:46,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:43:46,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:48,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:49,576 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:43:50,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:51,627 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:43:52,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:54,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:54,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:43:55,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:43:55,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:43:56,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:43:57,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:43:58,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:00,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:02,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:02,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:44:04,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:06,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:07,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:44:08,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:08,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:44:08,744 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:44:08,744 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:44:08,747 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:44:09,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:44:09,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:44:10,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:44:10,164 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:44:10,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:12,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:13,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:44:14,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:16,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:18,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:18,746 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:44:19,588 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:44:20,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:22,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:24,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:44:24,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:24,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:44:25,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:44:25,164 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:44:26,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:28,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:29,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:44:30,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:32,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:34,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:34,961 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:44:36,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:38,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:39,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:44:40,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:44:40,164 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:44:40,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:44:40,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:42,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:44,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:45,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:44:46,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:48,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:49,580 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:44:50,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:51,202 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:44:52,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:53,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:44:53,851 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:44:53,852 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:44:53,853 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:44:54,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:54,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:44:54,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:44:55,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:44:55,164 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:44:56,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:44:56,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:44:58,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:00,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:02,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:45:02,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:04,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:06,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:07,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:45:08,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:09,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:45:10,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:45:10,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:45:10,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:12,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:12,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:45:14,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:16,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:18,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:45:18,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:19,582 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:45:20,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:22,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:23,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:45:24,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:24,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:45:25,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:45:25,169 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:45:26,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:28,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:29,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:45:30,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:32,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:34,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:34,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:45:36,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:38,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:38,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:45:38,911 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:45:38,912 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:45:38,913 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:45:39,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:45:39,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:45:39,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:45:40,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:45:40,169 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:45:40,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:42,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:44,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:45,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:45:46,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:48,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:49,585 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:45:50,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:50,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:45:52,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:54,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:54,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:45:55,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:45:55,209 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:45:56,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:45:56,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:45:58,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:00,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:01,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:46:02,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:04,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:06,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:06,977 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:46:08,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:09,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:46:10,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:46:10,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:46:10,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:12,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:46:12,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:14,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:16,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:17,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:46:18,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:19,598 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:46:20,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:22,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:23,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:46:24,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:46:24,041 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:46:24,042 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:46:24,043 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:46:24,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:24,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:46:24,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:46:25,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:46:25,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:46:26,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:28,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:28,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:46:30,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:32,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:34,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:46:34,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:36,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:38,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:39,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:46:39,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:46:40,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:46:40,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:46:40,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:42,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:44,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:44,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:46:46,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:48,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:49,589 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:46:50,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:50,590 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:46:52,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:54,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:54,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:46:55,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:46:55,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:46:56,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:46:56,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:46:58,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:00,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:01,853 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:47:02,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:04,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:06,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:07,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:47:08,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:09,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:47:09,108 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:47:09,108 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:47:09,110 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:47:09,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:47:09,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:47:10,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:47:10,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:47:10,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:12,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:12,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:47:14,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:16,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:18,123 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:47:18,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:19,591 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:47:20,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:22,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:23,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:47:24,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:24,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:47:25,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:47:25,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:47:26,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:28,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:28,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:47:30,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:32,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:34,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:47:34,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:36,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:38,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:39,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:47:39,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:47:40,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:47:40,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:47:40,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:42,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:44,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:45,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:47:46,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:48,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:49,593 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:47:50,568 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:47:50,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:52,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:54,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:47:54,203 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:47:54,203 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:47:54,205 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:47:54,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:54,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:47:54,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:47:55,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:47:55,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:47:55,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:47:55,976 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:47:58,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:00,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:01,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:48:02,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:04,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:06,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:06,821 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:48:08,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:09,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:48:10,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:48:10,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:48:10,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:12,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:48:12,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:14,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:16,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:17,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:48:18,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:19,606 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:48:20,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:22,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:23,056 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:48:24,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:24,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:48:25,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:48:25,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:48:26,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:28,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:48:28,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:30,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:32,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:33,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:48:34,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:36,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:38,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:39,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:48:39,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:48:39,304 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:48:39,304 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:48:39,306 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:48:39,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:39,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:48:39,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:48:40,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:48:40,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:48:42,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:44,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:48:44,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:46,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:48,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:49,598 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:48:50,139 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:48:50,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:52,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:54,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:54,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:48:55,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:48:55,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:48:55,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:48:56,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:48:58,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:00,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:00,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:49:02,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:04,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:06,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:49:06,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:08,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:09,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:49:10,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:49:10,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:49:10,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:11,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:49:12,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:14,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:16,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:17,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:49:18,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:19,600 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:49:20,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:22,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:49:22,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:24,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:49:24,426 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:49:24,427 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:49:24,428 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:49:24,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:24,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:49:24,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:49:25,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:49:25,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:49:25,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:28,014 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:49:28,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:30,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:32,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:33,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:49:34,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:36,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:38,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:38,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:49:39,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:49:40,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:49:40,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:49:40,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:42,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:44,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:49:44,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:46,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:48,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:49,603 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:49:49,604 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:49:50,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:52,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:54,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:54,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:49:55,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:49:55,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:49:55,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:49:56,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:49:58,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:00,606 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:50:00,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:02,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:04,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:06,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:50:06,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:08,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:09,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:50:09,675 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:50:09,675 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:50:09,677 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:50:09,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:50:09,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:50:10,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:50:10,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:50:10,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:11,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:50:11,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:12,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:14,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:16,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:16,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:50:18,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:19,616 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:50:20,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:22,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:50:22,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:24,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:24,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:50:25,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:50:25,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:50:26,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:27,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:50:28,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:30,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:32,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:33,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:50:34,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:36,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:38,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:50:38,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:39,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:50:40,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:50:40,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:50:40,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:42,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:44,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:50:44,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:46,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:48,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:49,476 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:50:49,607 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:50:50,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:52,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:54,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:50:54,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:50:54,914 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:50:54,915 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:50:54,916 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:50:54,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:54,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:50:54,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:50:55,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:50:55,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:50:55,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:56,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:50:58,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:00,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:51:00,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:02,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:04,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:05,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:51:06,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:08,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:09,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:51:10,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:51:10,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:51:10,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:11,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:51:12,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:14,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:16,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:51:16,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:18,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:19,609 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:51:20,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:22,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:51:22,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:24,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:24,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:51:25,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:51:25,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:51:26,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:27,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:51:28,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:30,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:32,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:51:32,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:34,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:36,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:38,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:51:38,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:39,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:51:40,098 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:51:40,099 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:51:40,099 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:51:40,101 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:51:40,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:51:40,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:51:40,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:40,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:51:41,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:42,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:43,663 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:51:44,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:46,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:48,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:49,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:51:49,611 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:51:50,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:52,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:54,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:51:54,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:51:54,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:55,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:51:55,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:51:56,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:58,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:51:59,932 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:52:00,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:03,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:05,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:05,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:52:07,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:09,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:09,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:52:10,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:52:10,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:52:10,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:52:11,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:13,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:15,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:16,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:52:17,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:19,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:19,625 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:52:21,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:21,682 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:52:23,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:24,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:52:25,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:25,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:52:25,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:52:25,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:52:25,318 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:52:25,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:52:25,319 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:52:26,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:26,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:52:27,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:27,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:52:29,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:31,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:32,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:52:33,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:35,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:37,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:37,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:52:39,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:39,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:52:40,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:52:40,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:52:41,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:43,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:43,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:52:45,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:47,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:48,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:52:49,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:49,615 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:52:51,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:53,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:54,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:54,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:52:54,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:52:55,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:52:55,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:52:56,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:58,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:52:59,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:53:00,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:02,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:04,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:05,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:53:06,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:08,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:09,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:53:10,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:10,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:53:10,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:53:10,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:53:10,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:53:10,488 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:53:10,489 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:53:10,491 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:53:11,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:53:12,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:14,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:15,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:53:16,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:18,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:19,617 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:53:20,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:21,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:53:22,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:24,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:24,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:53:25,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:53:25,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:53:26,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:26,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:53:28,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:30,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:32,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:32,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:53:34,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:36,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:37,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:53:38,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:39,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:53:40,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:40,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:53:40,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:53:42,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:42,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:53:44,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:46,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:48,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:48,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:53:49,620 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:53:50,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:52,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:53,815 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:53:54,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:54,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:53:55,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:53:55,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:53:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:53:55,654 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:53:55,655 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:53:55,657 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:53:56,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:56,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:53:58,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:53:59,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:54:00,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:02,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:04,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:04,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:54:06,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:08,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:09,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:54:10,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:54:10,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:10,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:54:10,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:54:12,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:14,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:15,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:54:16,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:18,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:19,622 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:54:20,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:20,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:54:22,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:24,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:24,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:54:25,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:54:25,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:54:26,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:26,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:54:28,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:30,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:31,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:54:32,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:34,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:36,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:37,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:54:38,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:39,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:54:40,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:40,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:54:40,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:54:40,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:54:40,893 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:54:40,893 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:54:40,896 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:54:41,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:54:42,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:42,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:54:44,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:46,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:48,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:54:48,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:49,624 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:54:50,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:52,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:53,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:54:54,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:54,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:54:55,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:54:55,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:54:56,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:58,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:54:58,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:55:00,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:02,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:04,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:04,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:55:06,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:08,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:09,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:55:09,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:55:10,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:55:10,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:55:10,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:12,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:14,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:15,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:55:16,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:18,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:19,626 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:55:20,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:20,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:55:22,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:24,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:24,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:55:25,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:55:25,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:55:26,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:55:26,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:55:26,033 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:55:26,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:55:26,035 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:55:26,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:26,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:55:28,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:30,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:31,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:55:32,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:34,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:36,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:36,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:55:38,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:39,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:55:40,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:55:40,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:55:40,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:42,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:55:42,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:44,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:46,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:47,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:55:48,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:49,628 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:55:50,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:52,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:53,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:55:54,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:54,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:55:55,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:55:55,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:55:56,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:58,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:55:58,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:56:00,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:02,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:03,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:56:04,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:06,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:08,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:09,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:56:09,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:56:10,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:56:10,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:56:10,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:11,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:56:11,064 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:56:11,064 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:56:11,066 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:56:11,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:56:12,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:14,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:14,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:56:16,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:18,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:19,631 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:56:20,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:56:20,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:22,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:24,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:24,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:56:25,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:56:25,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:56:25,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:56:26,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:28,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:30,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:30,919 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:56:32,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:34,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:36,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:36,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:56:38,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:40,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:56:40,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:56:40,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:56:40,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:41,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:56:42,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:44,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:46,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:47,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:56:48,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:49,633 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:56:50,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:52,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:52,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:56:54,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:55,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:56:55,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:56:55,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:56:56,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:56:56,242 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:56:56,242 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:56:56,244 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:56:56,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:56:56,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:56:58,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:56:58,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:00,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:02,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:03,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:57:04,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:06,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:08,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:08,851 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:57:10,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:57:10,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:57:10,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:57:10,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:12,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:14,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:57:14,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:16,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:18,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:19,636 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:57:19,637 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:57:20,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:22,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:24,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:25,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:57:25,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:57:25,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:57:25,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:57:26,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:28,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:30,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:30,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:57:32,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:34,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:35,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:57:36,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:38,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:40,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:57:40,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:57:40,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:57:40,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:41,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:57:41,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:57:41,359 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:57:41,359 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:57:41,360 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:57:41,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:57:42,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:44,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:46,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:46,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:57:48,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:49,639 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:57:50,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:52,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:57:52,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:54,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:55,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:57:55,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:57:55,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:57:56,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:57:57,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:57:58,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:00,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:02,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:03,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:58:04,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:06,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:08,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:58:08,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:10,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:58:10,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:58:10,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:58:10,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:12,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:13,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:58:14,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:16,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:18,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:19,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:58:19,641 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:58:20,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:22,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:24,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:24,811 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:58:25,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:58:25,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:58:25,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:58:26,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:26,655 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:58:26,656 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:58:26,657 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:58:26,658 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:58:27,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:58:28,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:30,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:58:30,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:32,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:34,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:35,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:58:36,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:38,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:40,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:58:40,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:58:40,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:58:40,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:41,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:58:42,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:44,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:46,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:58:46,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:48,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:49,643 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:58:50,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:51,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:58:52,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:54,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:55,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:58:55,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:58:55,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:58:56,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:58:57,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:58:58,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:00,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:02,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:02,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:59:04,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:06,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:08,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:59:08,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:10,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:59:10,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:59:10,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:59:10,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:11,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:59:11,863 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:59:11,863 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:59:11,864 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:59:12,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:12,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:59:13,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:59:14,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:16,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:18,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:19,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:59:19,645 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:59:20,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:22,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:24,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:59:24,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:25,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:59:25,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:59:25,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:59:26,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:28,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:29,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:59:30,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:32,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:34,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:35,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:59:36,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:38,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:40,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:59:40,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:59:40,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:59:40,239 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:59:40,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:42,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:44,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:45,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:59:46,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:48,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:49,647 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 16:59:50,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:59:50,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:52,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:54,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:55,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 16:59:55,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 16:59:55,213 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 16:59:56,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 16:59:56,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 16:59:56,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 16:59:56,945 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 16:59:56,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 16:59:56,947 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 16:59:57,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 16:59:58,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:00,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:01,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:00:02,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:04,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:06,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:06,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:00:08,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:10,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:00:10,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:00:10,193 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:00:10,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:12,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:00:12,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:14,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:16,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:17,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:00:18,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:19,650 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:00:20,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:22,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:23,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:00:24,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:25,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:00:25,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:00:25,193 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:00:26,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:28,536 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:00:28,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:30,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:32,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:33,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:00:34,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:36,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:38,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:39,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:00:40,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:00:40,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:00:40,225 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:00:40,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:42,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:00:42,026 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:00:42,026 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:00:42,028 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:00:42,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:42,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:00:43,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:44,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:00:44,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:46,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:48,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:49,652 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:00:50,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:00:50,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:52,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:54,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:55,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:00:55,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:00:55,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:00:56,356 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:00:56,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:00:58,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:00,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:01,853 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:01:02,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:04,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:06,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:07,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:01:08,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:10,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:01:10,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:01:10,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:01:10,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:12,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:01:12,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:14,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:16,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:18,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:01:18,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:19,654 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:01:20,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:22,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:23,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:01:24,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:25,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:01:25,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:01:25,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:01:26,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:27,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:01:27,116 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:01:27,116 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:01:27,118 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:01:27,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:27,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:01:28,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:28,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:01:30,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:32,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:34,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:01:34,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:36,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:38,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:39,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:01:40,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:01:40,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:01:40,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:01:40,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:42,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:44,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:45,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:01:46,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:48,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:49,656 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:01:50,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:01:50,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:52,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:54,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:55,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:01:55,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:01:55,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:01:56,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:01:56,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:01:58,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:00,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:01,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:02:02,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:04,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:06,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:02:06,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:08,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:10,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:02:10,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:02:10,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:02:10,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:12,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:02:12,228 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:02:12,230 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:02:12,230 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:02:12,232 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:02:12,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:12,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:02:13,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:14,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:16,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:17,592 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:02:18,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:19,659 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:02:20,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:22,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:22,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:02:24,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:25,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:02:25,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:02:25,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:02:26,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:28,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:02:28,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:30,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:32,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:33,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:02:34,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:36,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:38,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:39,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:02:40,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:02:40,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:02:40,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:02:40,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:42,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:44,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:02:44,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:46,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:48,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:49,672 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:02:49,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:02:50,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:52,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:54,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:55,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:02:55,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:02:55,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:02:55,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:02:56,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:57,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:02:57,266 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:02:57,266 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:02:57,267 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:02:57,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:02:57,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:02:58,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:00,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:03:00,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:02,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:04,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:06,233 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:03:06,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:08,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:10,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:03:10,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:03:10,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:03:10,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:11,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:03:12,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:14,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:16,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:16,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:03:18,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:19,663 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:03:20,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:22,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:03:22,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:24,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:25,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:03:25,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:03:25,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:03:26,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:27,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:03:28,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:30,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:32,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:33,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:03:34,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:36,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:38,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:03:38,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:40,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:03:40,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:03:40,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:03:40,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:42,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:03:42,250 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:03:42,250 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:03:42,252 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:03:42,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:42,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:03:43,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:44,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:03:44,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:46,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:48,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:49,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:03:49,665 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:03:50,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:52,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:54,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:03:54,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:55,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:03:55,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:03:55,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:03:56,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:03:58,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:00,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:04:00,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:02,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:05,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:05,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:04:07,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:09,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:10,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:04:10,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:04:10,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:04:11,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:11,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:04:13,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:15,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:16,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:04:17,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:19,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:19,668 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:04:21,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:21,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:04:23,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:25,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:04:25,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:25,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:04:25,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:04:27,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:27,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:04:27,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:04:27,312 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:04:27,312 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:04:27,314 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:04:28,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:28,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:04:29,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:31,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:32,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:04:33,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:35,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:37,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:38,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:04:39,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:40,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:04:40,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:04:40,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:04:41,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:43,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:43,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:04:45,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:47,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:48,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:04:49,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:49,680 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:04:51,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:53,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:54,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:04:55,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:04:55,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:55,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:04:55,196 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:04:57,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:59,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:04:59,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:05:01,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:03,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:05,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:05,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:05:07,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:09,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:10,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:05:10,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:05:10,196 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:05:10,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:05:11,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:12,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:05:12,351 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:05:12,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:05:12,354 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:05:13,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:13,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:05:14,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:15,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:15,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:05:17,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:19,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:19,671 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:05:21,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:21,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:05:23,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:25,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:05:25,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:25,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:05:25,196 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:05:26,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:05:27,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:29,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:31,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:32,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:05:33,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:35,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:37,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:37,517 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:05:39,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:40,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:05:40,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:05:40,196 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:05:41,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:42,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:05:43,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:45,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:47,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:48,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:05:49,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:49,674 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:05:51,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:53,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:53,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:05:55,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:05:55,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:55,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:05:55,196 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:05:57,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:57,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:05:57,440 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:05:57,440 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:05:57,441 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:05:58,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:58,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:05:59,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:05:59,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:06:01,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:03,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:04,664 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:06:05,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:07,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:09,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:10,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:06:10,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:06:10,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:06:10,221 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:06:11,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:13,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:15,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:15,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:06:17,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:19,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:19,677 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:06:20,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:06:21,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:23,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:25,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:06:25,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:25,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:06:25,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:06:26,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:06:27,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:29,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:31,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:31,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:06:33,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:35,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:37,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:37,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:06:39,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:40,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:06:40,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:06:40,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:06:41,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:42,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:06:42,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:06:42,753 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:06:42,753 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:06:42,756 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:06:43,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:43,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:06:44,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:45,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:47,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:48,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:06:49,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:49,689 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:06:51,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:53,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:53,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:06:55,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:06:55,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:55,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:06:55,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:06:57,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:06:59,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:06:59,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:01,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:03,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:04,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:07:05,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:07,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:09,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:09,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:07:10,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:07:10,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:07:10,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:07:11,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:13,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:15,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:15,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:07:17,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:19,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:19,681 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:07:20,682 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:07:21,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:23,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:25,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:07:25,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:07:25,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:07:25,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:26,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:07:27,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:28,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:07:28,186 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:07:28,187 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:07:28,188 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:07:28,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:07:29,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:30,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:31,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:31,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:07:32,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:34,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:36,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:37,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:07:38,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:40,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:07:40,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:07:40,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:07:40,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:42,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:42,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:07:44,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:46,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:48,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:07:48,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:49,683 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:07:50,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:52,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:53,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:07:54,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:55,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:07:55,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:07:55,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:07:56,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:58,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:07:58,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:08:00,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:02,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:04,363 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:08:04,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:06,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:08,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:09,815 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:08:10,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:08:10,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:08:10,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:08:10,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:12,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:13,476 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:08:13,478 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:08:13,478 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:08:13,480 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:08:14,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:14,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:08:15,262 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:08:16,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:18,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:19,685 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:08:20,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:20,686 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:08:22,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:24,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:25,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:08:25,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:08:25,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:08:26,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:08:26,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:28,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:30,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:31,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:08:32,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:34,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:36,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:36,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:08:38,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:40,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:08:40,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:08:40,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:08:40,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:42,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:08:42,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:44,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:46,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:47,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:08:48,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:49,698 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:08:50,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:52,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:53,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:08:54,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:55,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:08:55,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:08:55,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:08:56,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:58,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:08:58,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:08:58,690 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:08:58,692 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:08:58,692 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:08:58,694 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:08:59,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:09:00,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:02,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:04,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:09:04,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:06,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:08,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:09,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:09:10,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:09:10,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:09:10,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:09:10,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:12,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:14,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:14,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:09:16,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:18,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:19,690 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:09:20,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:09:20,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:22,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:24,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:25,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:09:25,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:09:25,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:09:25,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:09:26,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:28,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:30,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:09:30,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:32,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:34,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:35,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:09:36,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:38,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:40,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:09:40,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:09:40,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:09:40,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:41,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:09:42,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:43,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:09:43,935 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:09:43,936 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:09:43,937 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:09:44,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:44,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:09:46,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:46,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:09:48,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:49,692 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:09:50,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:52,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:09:52,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:54,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:55,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:09:55,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:09:55,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:09:56,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:09:57,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:09:58,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:00,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:02,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:02,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:10:04,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:06,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:08,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:10:08,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:10,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:10:10,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:10:10,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:10:10,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:12,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:13,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:10:14,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:16,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:18,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:19,231 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:10:19,694 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:10:20,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:22,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:24,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:24,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:10:25,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:10:25,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:10:25,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:10:26,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:28,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:29,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:10:29,132 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:10:29,133 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:10:29,134 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:10:29,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:10:30,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:10:30,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:32,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:34,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:35,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:10:36,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:38,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:40,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:10:40,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:10:40,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:10:40,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:41,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:10:42,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:44,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:46,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:47,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:10:48,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:49,697 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:10:50,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:52,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:10:52,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:54,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:55,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:10:55,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:10:55,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:10:56,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:10:57,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:10:58,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:00,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:02,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:03,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:11:04,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:06,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:08,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:08,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:11:10,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:11:10,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:11:10,216 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:11:10,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:12,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:14,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:11:14,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:11:14,277 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:11:14,278 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:11:14,279 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:11:14,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:14,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:11:16,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:18,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:19,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:11:19,698 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:11:20,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:22,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:24,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:25,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:11:25,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:11:25,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:11:25,216 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:11:26,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:28,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:30,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:11:30,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:32,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:34,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:35,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:11:36,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:38,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:40,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:11:40,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:11:40,216 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:11:40,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:41,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:11:42,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:44,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:46,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:46,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:11:48,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:49,701 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:11:50,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:52,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:11:52,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:54,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:55,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:11:55,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:11:55,216 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:11:56,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:57,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:11:58,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:11:59,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:11:59,430 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:11:59,430 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:11:59,432 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:11:59,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:12:00,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:02,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:03,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:12:04,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:06,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:08,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:12:08,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:10,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:12:10,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:12:10,288 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:12:10,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:12,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:13,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:12:14,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:16,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:18,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:19,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:12:19,704 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:12:20,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:22,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:24,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:12:24,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:25,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:12:25,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:12:25,268 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:12:26,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:28,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:30,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:12:30,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:32,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:34,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:35,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:12:36,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:38,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:40,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:12:40,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:12:40,268 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:12:40,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:40,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:12:42,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:44,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:12:44,531 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:12:44,531 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:12:44,534 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:12:44,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:44,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:12:46,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:12:46,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:48,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:49,706 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:12:50,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:51,715 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:12:52,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:54,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:55,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:12:55,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:12:55,268 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:12:56,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:12:57,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:12:58,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:00,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:02,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:13:02,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:04,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:06,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:07,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:13:08,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:10,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:13:10,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:13:10,268 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:13:10,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:12,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:13,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:13:14,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:16,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:18,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:13:18,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:19,708 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:13:20,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:22,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:24,192 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:13:24,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:25,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:13:25,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:13:25,268 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:13:26,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:28,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:29,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:13:29,632 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:13:29,634 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:13:29,634 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:13:29,636 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:13:29,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:13:30,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:32,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:34,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:35,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:13:36,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:38,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:40,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:13:40,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:13:40,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:13:40,268 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:13:40,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:42,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:44,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:45,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:13:46,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:48,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:49,710 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:13:50,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:51,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:13:52,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:54,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:55,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:13:55,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:13:55,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:13:56,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:13:56,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:13:58,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:00,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:02,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:14:02,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:04,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:06,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:07,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:14:08,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:10,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:14:10,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:14:10,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:14:10,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:12,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:12,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:14:14,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:14:14,746 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:14:14,746 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:14:14,748 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:14:14,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:14,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:14:16,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:18,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:14:18,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:19,712 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:14:20,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:22,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:23,762 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:14:24,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:25,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:14:25,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:14:25,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:14:26,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:28,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:29,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:14:30,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:32,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:34,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:14:34,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:36,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:38,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:39,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:14:40,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:14:40,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:14:40,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:14:40,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:42,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:44,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:45,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:14:46,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:48,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:49,715 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:14:50,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:14:50,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:52,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:54,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:55,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:14:55,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:14:55,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:14:56,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:14:56,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:58,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:14:59,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:14:59,826 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:14:59,826 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:14:59,828 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:14:59,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:15:00,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:01,597 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:15:01,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:02,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:04,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:06,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:06,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:15:08,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:10,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:15:10,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:15:10,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:15:10,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:12,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:15:12,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:14,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:16,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:17,808 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:15:18,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:19,717 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:15:20,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:22,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:23,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:15:24,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:25,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:15:25,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:15:25,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:15:26,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:28,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:15:28,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:30,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:32,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:34,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:15:34,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:36,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:38,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:39,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:15:40,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:15:40,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:15:40,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:15:40,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:42,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:44,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:15:44,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:15:44,874 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:15:44,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:15:44,878 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:15:44,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:44,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:15:45,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:46,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:48,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:49,719 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:15:50,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:15:50,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:52,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:54,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:55,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:15:55,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:15:55,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:15:55,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:15:56,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:15:58,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:00,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:00,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:16:02,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:04,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:06,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:16:06,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:08,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:10,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:16:10,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:16:10,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:16:10,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:11,766 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:16:12,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:14,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:16,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:17,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:16:18,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:19,721 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:16:20,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:22,553 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:16:22,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:24,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:25,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:16:25,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:16:25,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:16:26,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:27,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:16:28,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:29,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:16:29,805 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:16:29,806 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:16:29,808 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:16:29,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:16:30,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:31,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:32,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:33,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:16:34,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:36,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:38,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:16:39,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:40,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:16:40,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:16:40,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:16:41,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:43,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:44,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:16:45,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:47,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:49,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:49,594 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:16:49,724 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:16:51,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:53,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:55,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:55,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:16:55,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:16:55,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:16:55,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:16:57,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:16:59,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:00,476 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:17:01,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:03,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:05,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:05,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:17:07,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:09,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:10,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:17:10,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:17:10,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:17:11,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:11,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:17:13,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:14,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:17:14,902 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:17:14,902 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:17:14,904 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:17:15,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:15,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:17:16,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:16,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:17:17,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:19,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:19,726 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:17:21,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:22,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:17:23,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:25,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:25,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:17:25,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:17:25,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:17:27,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:27,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:17:29,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:31,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:32,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:17:33,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:35,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:37,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:38,243 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:17:39,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:40,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:17:40,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:17:40,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:17:41,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:43,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:43,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:17:45,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:47,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:49,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:17:49,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:49,728 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:17:51,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:53,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:54,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:17:55,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:17:55,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:55,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:17:55,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:17:57,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:59,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:17:59,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:17:59,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:17:59,920 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:17:59,920 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:17:59,921 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:18:00,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:18:01,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:02,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:03,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:05,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:05,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:18:07,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:09,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:10,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:18:10,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:18:10,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:18:10,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:18:11,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:13,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:15,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:16,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:18:17,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:19,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:19,729 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:18:21,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:21,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:18:23,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:25,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:18:25,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:18:25,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:18:25,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:26,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:18:27,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:29,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:31,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:32,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:18:33,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:35,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:37,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:37,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:18:39,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:40,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:18:40,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:18:40,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:18:41,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:42,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:18:43,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:44,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:18:44,804 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:18:44,805 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:18:44,807 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:18:45,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:45,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:18:46,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:47,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:48,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:18:49,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:49,732 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:18:51,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:53,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:53,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:18:55,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:18:55,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:18:55,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:18:55,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:57,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:18:59,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:18:59,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:01,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:03,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:04,605 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:19:05,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:07,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:09,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:09,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:19:10,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:19:10,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:19:10,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:19:11,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:13,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:15,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:15,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:19:17,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:19,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:19,746 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:19:20,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:19:21,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:23,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:25,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:19:25,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:19:25,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:19:25,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:26,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:19:27,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:29,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:29,815 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:19:29,817 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:19:29,817 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:19:29,819 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:19:30,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:19:31,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:31,592 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:19:32,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:33,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:35,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:37,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:19:37,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:39,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:40,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:19:40,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:19:40,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:19:41,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:42,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:19:43,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:45,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:47,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:47,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:19:49,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:49,737 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:19:51,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:53,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:19:53,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:55,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:19:55,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:19:55,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:19:55,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:57,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:19:58,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:19:59,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:01,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:03,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:03,932 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:20:05,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:07,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:09,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:20:09,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:10,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:20:10,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:20:10,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:20:11,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:13,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:14,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:20:14,729 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:20:14,731 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:20:14,731 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:20:14,733 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:20:15,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:15,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:20:16,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:17,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:19,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:19,739 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:20:19,739 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:20:21,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:23,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:25,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:20:25,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:20:25,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:20:25,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:20:25,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:27,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:29,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:30,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:20:31,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:33,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:35,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:36,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:20:37,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:39,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:40,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:20:40,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:20:40,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:20:41,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:41,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:20:43,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:45,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:47,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:20:47,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:49,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:49,741 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:20:51,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:52,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:20:53,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:55,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:20:55,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:20:55,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:20:55,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:57,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:58,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:20:59,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:20:59,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:20:59,905 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:20:59,906 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:20:59,907 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:21:00,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:21:01,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:02,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:03,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:03,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:21:05,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:07,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:08,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:21:09,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:10,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:21:10,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:21:10,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:21:11,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:13,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:14,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:21:15,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:17,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:19,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:19,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:21:19,755 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:21:21,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:23,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:25,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:21:25,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:21:25,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:21:25,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:21:25,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:27,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:29,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:30,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:21:31,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:33,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:35,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:35,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:21:37,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:39,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:40,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:21:40,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:21:40,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:21:41,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:21:41,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:43,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:45,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:21:45,014 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:21:45,014 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:21:45,016 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:21:45,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:45,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:21:46,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:46,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:21:47,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:49,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:49,745 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:21:51,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:52,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:21:53,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:55,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:21:55,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:21:55,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:21:55,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:57,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:21:57,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:21:59,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:01,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:03,014 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:22:03,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:05,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:07,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:08,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:22:09,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:10,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:22:10,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:22:10,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:22:11,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:13,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:13,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:22:15,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:17,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:19,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:22:19,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:19,747 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:22:21,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:23,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:24,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:22:25,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:22:25,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:22:25,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:22:25,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:27,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:29,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:30,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:22:30,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:22:30,164 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:22:30,165 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:22:30,166 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:22:30,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:22:31,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:32,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:33,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:35,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:35,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:22:37,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:39,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:40,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:22:40,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:22:40,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:22:40,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:22:41,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:43,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:45,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:46,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:22:47,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:49,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:49,749 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:22:51,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:51,794 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:22:53,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:55,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:22:55,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:22:55,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:22:55,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:57,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:22:57,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:22:59,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:01,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:02,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:23:03,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:05,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:06,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:07,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:23:08,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:10,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:23:10,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:23:10,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:23:10,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:12,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:13,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:23:14,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:15,239 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:23:15,240 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:23:15,241 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:23:15,242 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:23:15,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:23:16,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:18,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:18,810 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:23:19,762 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:23:20,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:22,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:24,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:23:24,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:25,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:23:25,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:23:25,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:23:26,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:28,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:29,582 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:23:30,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:32,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:34,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:34,976 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:23:36,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:38,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:40,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:23:40,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:23:40,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:23:40,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:23:40,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:42,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:44,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:45,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:23:46,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:48,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:49,754 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:23:50,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:51,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:23:52,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:54,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:55,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:23:55,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:23:55,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:23:56,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:23:56,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:23:58,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:00,239 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:24:00,241 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:24:00,241 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:24:00,243 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:24:00,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:00,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:24:02,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:24:02,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:04,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:06,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:07,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:24:08,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:10,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:24:10,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:24:10,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:24:10,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:12,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:12,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:24:14,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:16,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:18,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:24:18,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:19,757 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:24:20,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:22,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:23,567 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:24:24,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:25,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:24:25,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:24:25,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:24:26,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:28,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:28,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:24:30,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:32,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:34,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:24:34,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:36,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:38,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:39,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:24:40,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:24:40,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:24:40,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:24:40,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:42,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:44,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:45,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:24:45,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:24:45,248 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:24:45,248 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:24:45,249 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:24:45,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:24:46,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:48,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:49,759 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:24:50,627 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:24:50,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:52,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:54,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:55,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:24:55,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:24:55,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:24:56,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:24:56,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:24:58,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:00,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:01,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:25:02,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:04,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:06,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:06,811 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:25:08,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:10,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:25:10,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:25:10,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:25:10,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:12,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:25:12,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:14,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:16,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:17,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:25:18,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:19,770 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:25:20,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:22,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:23,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:25:24,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:25,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:25:25,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:25:25,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:25:26,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:28,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:25:28,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:30,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:25:30,260 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:25:30,261 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:25:30,262 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:25:30,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:30,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:25:32,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:33,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:25:34,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:36,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:38,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:39,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:25:40,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:25:40,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:25:40,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:25:40,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:42,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:44,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:25:44,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:46,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:48,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:49,763 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:25:49,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:25:50,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:52,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:54,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:54,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:25:55,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:25:55,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:25:55,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:25:56,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:25:58,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:00,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:26:00,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:02,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:04,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:05,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:26:06,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:08,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:10,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:26:10,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:26:10,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:26:10,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:11,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:26:12,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:14,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:15,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:26:15,432 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:26:15,432 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:26:15,435 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:26:15,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:26:16,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:26:16,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:18,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:19,765 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:26:20,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:21,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:26:22,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:24,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:25,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:26:25,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:26:25,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:26:26,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:27,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:26:28,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:30,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:32,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:26:32,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:34,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:36,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:38,065 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:26:38,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:40,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:26:40,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:26:40,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:26:40,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:42,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:43,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:26:44,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:46,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:48,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:48,851 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:26:49,767 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:26:50,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:52,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:54,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:26:54,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:55,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:26:55,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:26:55,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:26:56,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:58,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:26:59,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:27:00,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:27:00,503 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:27:00,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:27:00,506 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:27:00,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:00,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:27:02,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:04,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:05,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:27:06,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:08,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:10,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:27:10,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:27:10,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:27:10,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:27:10,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:12,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:14,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:15,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:27:16,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:18,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:19,780 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:27:20,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:21,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:27:22,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:24,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:25,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:27:25,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:27:25,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:27:26,706 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:27:26,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:28,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:30,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:32,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:27:32,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:34,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:36,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:37,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:27:38,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:40,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:27:40,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:27:40,275 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:27:40,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:42,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:27:43,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:45,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:45,604 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:27:45,606 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:27:45,607 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:27:45,608 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:27:46,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:46,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:27:47,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:48,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:27:49,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:49,772 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:27:51,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:53,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:53,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:27:55,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:55,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:27:55,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:27:55,275 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:27:57,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:59,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:27:59,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:28:01,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:03,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:04,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:28:05,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:07,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:09,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:09,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:28:10,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:28:10,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:28:10,275 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:28:11,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:13,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:15,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:15,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:28:17,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:19,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:19,774 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:28:20,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:28:21,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:23,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:25,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:25,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:28:25,288 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:28:25,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:28:26,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:28:27,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:29,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:30,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:28:30,616 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:28:30,616 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:28:30,618 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:28:31,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:31,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:28:31,618 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:28:32,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:33,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:35,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:36,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:28:37,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:39,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:40,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:28:40,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:28:40,282 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:28:41,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:42,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:28:43,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:45,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:47,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:47,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:28:49,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:49,776 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:28:51,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:53,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:53,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:28:55,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:55,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:28:55,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:28:55,282 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:28:57,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:28:58,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:28:59,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:01,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:03,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:03,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:29:05,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:07,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:09,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:09,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:29:10,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:29:10,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:29:10,296 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:29:11,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:13,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:14,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:29:15,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:15,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:29:15,630 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:29:15,630 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:29:15,632 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:29:16,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:16,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:29:17,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:19,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:19,794 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:29:20,795 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:29:21,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:23,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:25,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:25,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:29:25,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:29:25,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:29:26,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:29:27,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:29,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:31,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:31,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:29:33,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:35,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:37,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:37,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:29:39,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:40,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:29:40,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:29:40,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:29:41,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:42,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:29:43,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:45,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:47,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:47,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:29:49,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:49,781 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:29:51,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:53,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:53,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:29:55,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:55,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:29:55,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:29:55,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:29:57,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:29:58,683 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:29:59,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:00,511 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:30:00,513 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:30:00,513 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:30:00,515 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:30:01,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:01,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:30:02,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:04,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:30:05,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:07,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:09,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:09,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:30:10,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:30:10,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:30:10,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:30:11,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:13,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:14,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:30:15,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:17,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:19,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:19,783 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:30:20,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:30:21,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:23,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:25,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:25,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:30:25,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:30:25,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:30:25,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:30:27,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:29,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:31,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:30:31,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:33,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:35,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:36,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:30:37,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:39,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:40,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:30:40,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:30:40,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:30:41,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:41,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:30:43,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:45,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:45,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:30:45,559 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:30:45,560 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:30:45,562 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:30:46,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:46,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:30:47,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:30:49,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:49,785 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:30:51,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:52,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:30:53,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:55,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:55,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:30:55,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:30:55,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:30:57,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:30:58,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:30:59,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:01,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:03,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:03,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:31:05,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:07,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:08,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:31:09,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:10,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:31:10,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:31:10,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:31:11,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:13,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:14,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:31:15,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:17,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:19,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:19,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:31:19,787 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:31:21,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:23,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:25,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:31:25,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:25,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:31:25,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:31:25,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:31:27,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:29,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:30,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:31:30,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:31:30,535 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:31:30,536 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:31:30,537 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:31:31,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:31,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:31:32,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:35,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:35,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:31:37,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:39,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:40,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:31:40,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:31:40,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:31:41,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:41,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:31:43,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:45,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:46,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:31:47,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:49,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:49,789 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:31:51,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:52,097 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:31:53,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:55,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:31:55,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:31:55,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:31:55,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:57,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:31:57,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:31:59,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:01,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:02,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:32:03,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:05,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:07,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:08,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:32:09,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:10,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:32:10,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:32:10,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:32:11,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:13,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:13,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:32:15,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:15,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:32:15,493 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:32:15,493 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:32:15,495 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:32:16,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:16,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:32:19,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:32:19,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:19,791 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:32:21,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:23,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:24,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:32:25,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:32:25,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:32:25,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:32:25,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:27,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:29,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:29,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:32:31,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:33,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:35,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:32:35,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:37,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:39,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:40,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:32:40,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:32:40,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:32:40,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:32:41,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:43,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:45,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:46,125 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:32:47,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:49,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:49,794 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:32:51,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:51,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:32:53,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:55,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:32:55,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:32:55,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:32:55,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:56,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:32:57,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:32:59,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:00,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:33:00,527 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:33:00,527 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:33:00,529 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:33:01,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:01,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:33:02,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:33:02,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:05,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:07,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:07,688 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:33:09,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:10,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:33:10,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:33:10,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:33:11,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:13,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:33:13,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:15,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:17,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:18,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:33:19,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:19,796 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:33:21,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:23,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:23,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:33:25,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:33:25,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:33:25,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:33:25,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:27,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:29,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:33:29,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:31,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:33,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:34,597 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:33:35,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:37,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:39,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:40,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:33:40,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:33:40,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:33:40,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:33:41,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:43,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:45,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:33:45,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:33:45,376 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:33:45,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:33:45,378 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:33:45,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:45,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:33:46,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:49,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:49,798 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:33:50,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:33:51,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:53,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:55,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:33:55,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:33:55,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:33:55,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:56,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:33:57,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:33:59,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:01,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:01,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:34:03,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:05,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:06,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:34:07,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:09,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:10,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:34:10,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:34:10,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:34:11,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:12,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:34:13,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:15,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:17,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:17,721 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:34:19,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:19,801 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:34:21,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:23,117 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:34:23,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:25,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:34:25,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:34:25,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:34:25,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:27,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:28,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:34:29,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:30,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:34:30,334 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:34:30,335 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:34:30,337 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:34:30,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:34:31,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:32,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:33,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:34:35,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:37,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:39,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:34:39,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:40,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:34:40,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:34:40,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:34:41,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:43,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:44,711 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:34:45,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:47,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:49,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:49,803 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:34:49,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:34:51,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:53,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:55,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:34:55,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:34:55,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:34:55,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:34:55,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:57,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:34:59,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:00,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:35:01,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:03,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:05,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:06,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:35:07,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:09,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:10,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:35:10,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:35:10,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:35:11,431 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:35:11,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:13,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:15,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:35:15,339 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:35:15,339 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:35:15,340 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:35:15,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:15,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:35:16,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:17,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:35:19,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:19,805 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:35:21,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:22,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:35:23,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:25,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:35:25,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:35:25,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:35:25,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:27,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:27,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:35:29,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:31,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:33,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:35:33,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:35,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:37,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:38,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:35:39,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:40,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:35:40,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:35:40,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:35:41,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:43,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:44,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:35:45,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:47,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:49,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:35:49,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:49,807 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:35:51,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:53,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:54,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:35:55,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:35:55,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:35:55,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:35:55,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:57,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:35:59,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:00,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:36:00,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:36:00,333 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:36:00,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:36:00,335 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:36:00,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:36:01,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:02,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:05,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:05,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:36:07,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:09,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:10,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:36:10,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:36:10,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:36:11,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:36:11,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:13,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:15,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:16,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:36:17,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:19,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:19,809 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:36:21,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:21,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:36:23,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:25,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:36:25,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:36:25,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:36:25,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:27,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:36:27,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:29,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:31,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:32,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:36:33,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:35,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:37,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:37,975 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:36:39,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:40,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:36:40,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:36:40,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:36:41,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:43,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:36:43,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:45,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:36:45,172 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:36:45,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:36:45,176 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:36:45,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:45,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:36:46,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:48,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:36:49,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:49,815 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:36:51,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:53,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:54,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:36:55,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:36:55,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:36:55,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:36:55,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:57,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:36:59,517 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:36:59,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:01,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:03,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:04,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:37:05,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:07,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:09,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:10,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:37:10,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:37:10,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:37:10,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:37:11,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:13,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:15,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:15,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:37:17,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:19,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:19,817 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:37:20,819 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:37:21,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:23,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:25,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:37:25,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:37:25,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:37:25,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:26,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:37:27,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:29,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:30,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:37:30,054 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:37:30,054 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:37:30,055 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:37:30,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:37:31,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:31,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:37:32,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:35,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:37,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:37:37,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:39,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:40,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:37:40,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:37:40,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:37:41,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:42,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:37:42,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:44,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:46,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:47,941 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:37:48,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:49,819 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:37:50,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:52,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:53,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:37:54,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:55,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:37:55,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:37:55,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:37:56,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:37:58,758 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:37:58,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:00,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:02,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:04,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:38:04,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:06,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:08,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:09,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:38:10,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:38:10,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:38:10,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:38:10,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:12,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:14,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:15,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:38:15,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:38:15,035 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:38:15,036 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:38:15,037 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:38:15,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:38:16,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:18,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:19,822 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:38:20,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:38:20,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:22,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:24,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:25,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:38:25,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:38:25,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:38:25,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:38:26,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:28,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:30,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:31,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:38:33,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:35,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:36,512 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:38:37,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:39,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:40,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:38:40,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:38:40,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:38:41,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:41,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:38:43,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:45,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:47,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:47,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:38:49,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:49,824 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:38:51,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:52,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:38:53,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:55,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:55,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:38:55,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:38:55,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:38:57,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:58,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:38:59,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:38:59,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:38:59,824 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:38:59,824 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:38:59,826 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:39:00,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:39:01,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:03,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:03,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:39:05,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:07,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:08,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:39:09,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:10,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:39:10,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:39:10,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:39:11,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:13,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:14,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:39:15,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:17,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:19,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:19,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:39:19,826 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:39:21,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:23,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:24,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:39:25,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:25,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:39:25,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:39:25,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:39:27,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:29,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:30,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:39:31,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:33,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:35,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:35,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:39:37,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:39,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:40,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:39:40,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:39:40,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:39:41,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:41,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:39:43,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:44,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:39:44,720 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:39:44,720 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:39:44,722 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:39:45,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:45,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:39:46,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:39:47,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:49,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:49,829 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:39:51,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:51,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:39:53,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:55,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:55,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:39:55,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:39:55,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:39:57,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:39:57,363 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:39:59,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:01,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:02,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:40:03,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:05,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:07,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:08,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:40:09,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:10,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:40:10,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:40:10,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:40:11,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:13,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:13,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:40:15,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:17,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:18,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:40:19,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:19,831 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:40:21,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:23,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:24,235 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:40:25,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:25,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:40:25,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:40:25,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:40:27,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:29,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:29,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:40:29,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:40:29,662 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:40:29,662 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:40:29,664 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:40:30,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:30,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:40:33,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:35,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:40:35,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:37,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:39,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:40,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:40:40,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:40:40,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:40:40,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:40:41,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:43,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:45,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:45,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:40:47,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:49,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:49,833 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:40:50,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:40:51,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:53,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:55,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:55,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:40:55,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:40:55,294 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:40:56,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:40:57,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:40:59,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:01,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:01,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:41:03,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:05,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:07,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:07,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:41:09,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:10,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:41:10,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:41:10,294 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:41:11,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:12,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:41:13,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:14,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:41:14,476 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:41:14,476 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:41:14,478 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:41:15,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:15,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:41:16,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:18,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:41:19,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:19,835 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:41:21,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:23,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:23,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:41:25,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:25,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:41:25,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:41:25,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:41:27,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:28,807 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:41:29,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:31,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:33,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:34,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:41:35,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:37,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:39,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:39,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:41:40,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:41:40,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:41:40,300 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:41:41,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:43,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:45,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:41:45,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:47,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:49,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:49,848 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:41:50,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:41:51,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:53,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:55,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:55,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:41:55,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:41:55,300 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:41:55,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:41:57,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:59,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:41:59,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:41:59,393 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:41:59,393 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:41:59,395 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:42:00,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:00,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:42:01,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:42:03,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:05,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:06,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:42:07,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:09,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:10,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:42:10,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:42:10,300 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:42:11,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:11,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:42:13,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:15,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:17,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:17,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:42:19,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:19,839 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:42:21,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:22,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:42:23,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:25,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:25,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:42:25,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:42:25,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:42:27,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:28,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:42:29,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:31,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:33,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:33,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:42:35,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:37,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:38,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:42:39,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:40,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:42:40,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:42:40,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:42:41,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:43,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:44,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:42:44,275 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:42:44,277 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:42:44,277 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:42:44,280 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:42:44,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:42:45,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:46,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:49,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:49,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:42:49,841 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:42:51,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:53,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:55,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:42:55,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:42:55,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:42:55,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:42:55,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:57,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:42:59,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:00,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:43:01,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:03,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:05,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:05,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:43:07,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:09,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:10,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:43:10,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:43:10,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:43:11,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:43:11,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:13,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:15,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:16,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:43:17,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:19,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:19,881 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:43:21,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:21,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:43:23,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:25,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:43:25,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:43:25,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:43:25,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:27,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:43:27,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:29,096 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:43:29,097 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:43:29,098 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:43:29,100 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:43:29,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:29,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:43:30,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:32,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:43:33,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:35,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:37,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:38,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:43:39,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:40,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:43:40,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:43:40,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:43:41,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:43,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:43:43,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:45,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:47,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:48,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:43:49,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:49,893 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:43:51,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:53,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:54,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:43:55,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:43:55,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:43:55,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:43:55,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:57,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:59,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:43:59,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:44:01,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:03,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:04,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:44:05,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:07,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:09,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:10,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:44:10,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:44:10,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:44:10,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:44:11,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:13,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:13,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:44:13,987 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:44:13,987 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:44:13,989 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:44:14,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:44:15,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:15,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:44:16,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:19,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:19,886 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:44:20,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:44:21,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:23,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:25,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:44:25,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:44:25,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:44:25,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:26,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:44:27,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:29,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:31,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:31,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:44:33,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:35,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:37,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:44:37,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:39,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:40,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:44:40,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:44:40,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:44:41,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:42,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:44:43,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:45,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:47,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:48,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:44:49,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:49,888 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:44:51,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:53,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:53,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:44:55,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:44:55,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:44:55,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:44:55,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:57,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:59,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:44:59,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:44:59,043 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:44:59,043 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:44:59,045 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:44:59,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:44:59,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:45:00,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:03,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:04,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:45:05,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:07,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:09,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:09,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:45:10,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:45:10,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:45:10,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:45:11,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:13,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:15,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:45:15,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:17,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:19,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:19,891 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:45:20,635 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:45:21,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:23,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:25,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:45:25,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:45:25,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:45:25,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:26,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:45:27,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:29,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:31,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:31,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:45:33,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:35,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:36,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:45:37,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:39,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:40,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:45:40,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:45:40,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:45:41,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:42,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:45:43,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:44,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:45:44,203 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:45:44,203 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:45:44,205 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:45:44,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:45:45,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:46,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:47,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:45:49,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:49,905 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:45:51,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:53,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:45:53,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:55,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:45:55,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:45:55,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:45:55,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:57,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:45:58,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:45:59,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:01,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:03,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:04,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:46:05,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:07,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:09,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:09,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:46:10,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:46:10,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:46:10,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:46:11,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:13,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:15,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:46:15,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:17,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:19,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:19,895 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:46:20,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:46:21,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:23,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:25,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:46:25,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:46:25,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:46:25,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:46:25,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:27,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:29,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:46:29,448 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:46:29,448 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:46:29,450 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:46:29,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:29,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:46:30,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:31,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:46:31,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:33,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:35,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:36,618 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:46:37,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:39,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:40,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:46:40,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:46:40,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:46:41,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:42,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:46:43,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:45,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:47,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:46:47,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:49,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:49,897 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:46:51,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:52,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:46:53,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:55,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:46:55,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:46:55,323 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:46:55,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:57,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:46:58,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:46:59,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:01,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:03,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:03,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:47:05,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:07,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:09,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:47:09,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:10,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:47:10,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:47:10,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:47:11,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:13,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:14,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:47:14,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:47:14,625 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:47:14,626 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:47:14,629 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:47:15,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:15,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:47:16,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:17,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:19,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:19,901 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:47:19,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:47:21,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:23,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:25,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:47:25,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:47:25,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:47:25,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:47:25,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:27,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:29,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:30,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:47:31,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:33,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:35,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:36,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:47:37,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:39,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:40,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:47:40,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:47:40,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:47:41,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:47:41,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:43,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:45,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:47,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:47:47,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:49,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:49,915 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:47:51,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:52,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:47:53,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:55,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:47:55,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:47:55,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:47:55,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:57,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:47:57,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:59,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:47:59,720 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:47:59,720 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:47:59,722 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:47:59,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:47:59,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:48:00,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:01,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:03,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:48:03,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:05,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:07,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:08,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:48:09,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:10,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:48:10,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:48:10,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:48:11,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:13,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:14,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:48:15,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:17,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:19,466 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:48:19,906 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:48:19,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:21,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:23,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:24,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:48:25,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:48:25,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:48:25,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:48:25,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:27,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:29,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:30,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:48:31,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:33,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:35,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:48:35,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:37,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:39,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:40,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:48:40,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:48:40,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:48:41,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:48:41,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:43,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:44,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:48:44,706 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:48:44,707 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:48:44,709 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:48:44,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:48:45,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:46,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:48:46,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:47,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:49,908 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:48:49,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:51,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:48:51,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:53,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:55,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:48:55,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:48:55,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:48:55,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:57,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:48:57,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:48:59,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:01,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:02,725 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:49:03,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:05,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:07,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:08,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:49:09,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:10,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:49:10,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:49:10,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:49:11,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:13,519 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:49:13,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:15,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:17,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:18,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:49:19,910 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:49:19,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:22,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:24,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:24,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:49:25,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:49:25,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:49:25,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:49:26,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:28,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:29,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:49:29,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:49:29,751 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:49:29,751 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:49:29,753 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:49:30,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:30,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:49:31,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:32,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:34,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:35,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:49:36,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:38,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:40,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:40,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:49:40,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:49:40,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:49:40,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:49:42,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:44,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:45,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:49:46,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:48,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:49,929 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:49:50,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:50,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:49:52,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:54,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:55,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:49:55,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:49:55,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:49:56,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:49:56,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:49:58,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:00,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:02,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:02,056 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:50:04,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:06,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:07,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:50:08,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:10,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:10,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:50:10,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:50:10,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:50:12,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:12,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:50:14,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:14,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:50:14,526 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:50:14,526 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:50:14,528 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:50:15,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:50:16,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:17,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:18,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:50:18,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:19,915 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:50:20,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:22,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:23,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:50:24,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:25,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:50:25,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:50:25,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:50:26,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:28,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:28,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:50:30,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:32,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:34,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:50:34,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:36,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:38,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:39,432 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:50:40,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:40,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:50:40,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:50:40,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:50:42,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:44,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:44,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:50:45,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:47,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:49,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:49,917 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:50:49,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:50:51,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:53,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:55,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:55,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:50:55,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:50:55,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:50:55,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:50:57,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:59,046 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:50:59,047 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:50:59,048 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:50:59,049 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:50:59,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:50:59,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:51:00,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:51:01,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:03,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:05,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:06,117 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:51:07,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:09,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:10,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:51:10,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:51:10,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:51:11,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:11,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:51:13,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:15,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:16,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:51:17,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:19,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:19,919 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:51:21,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:22,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:51:23,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:25,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:25,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:51:25,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:51:25,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:51:27,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:27,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:51:29,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:31,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:32,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:51:33,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:35,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:37,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:38,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:51:39,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:40,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:51:40,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:51:40,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:51:41,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:43,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:43,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:51:43,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:51:43,559 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:51:43,559 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:51:43,560 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:51:44,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:51:45,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:47,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:48,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:51:49,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:49,932 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:51:51,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:53,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:54,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:51:55,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:55,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:51:55,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:51:55,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:51:57,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:59,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:51:59,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:52:01,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:03,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:04,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:52:05,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:07,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:09,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:10,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:52:10,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:52:10,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:52:10,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:52:11,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:13,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:15,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:15,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:52:17,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:19,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:19,924 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:52:20,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:52:21,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:23,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:25,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:25,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:52:25,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:52:25,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:52:26,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:52:27,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:28,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:52:28,024 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:52:28,024 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:52:28,026 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:52:28,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:52:29,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:30,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:31,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:31,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:52:33,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:35,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:36,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:52:37,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:39,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:40,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:52:40,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:52:40,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:52:41,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:42,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:52:43,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:45,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:47,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:47,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:52:49,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:49,926 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:52:51,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:52,976 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:52:53,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:55,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:55,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:52:55,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:52:55,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:52:57,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:52:58,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:52:59,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:01,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:03,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:03,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:53:05,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:07,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:09,043 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:53:09,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:10,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:53:10,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:53:10,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:53:11,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:12,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:53:12,627 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:53:12,627 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:53:12,629 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:53:13,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:13,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:53:14,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:14,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:53:15,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:17,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:19,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:19,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:53:19,928 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:53:21,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:23,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:25,046 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:53:25,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:25,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:53:25,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:53:25,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:53:27,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:29,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:30,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:53:31,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:33,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:35,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:35,770 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:53:37,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:39,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:40,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:53:40,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:53:40,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:53:41,123 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:53:41,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:43,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:45,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:46,470 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:53:47,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:49,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:49,931 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:53:51,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:51,805 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:53:53,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:55,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:55,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:53:55,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:53:55,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:53:57,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:53:57,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:53:57,194 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:53:57,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:53:57,196 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:53:57,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:57,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:53:58,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:53:59,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:01,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:02,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:54:03,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:05,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:07,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:07,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:54:09,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:10,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:54:10,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:54:10,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:54:11,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:13,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:54:13,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:15,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:17,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:18,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:54:19,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:19,932 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:54:21,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:23,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:23,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:54:25,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:54:25,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:25,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:54:25,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:54:27,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:29,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:54:29,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:31,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:33,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:34,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:54:35,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:37,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:39,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:39,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:54:40,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:54:40,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:54:40,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:54:41,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:41,662 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:54:41,664 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:54:41,664 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:54:41,666 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:54:42,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:42,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:54:43,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:45,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:54:45,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:47,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:49,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:49,934 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:54:50,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:54:51,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:53,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:55,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:54:55,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:54:55,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:54:55,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:56,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:54:57,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:54:59,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:01,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:02,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:55:03,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:05,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:07,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:07,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:55:09,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:10,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:55:10,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:55:10,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:55:11,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:12,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:55:13,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:15,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:17,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:18,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:55:19,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:19,937 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:55:21,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:23,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:23,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:55:25,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:55:25,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:55:25,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:55:25,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:26,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:55:26,299 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:55:26,300 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:55:26,301 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:55:26,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:55:27,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:28,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:29,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:55:29,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:31,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:33,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:34,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:55:35,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:37,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:39,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:39,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:55:40,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:55:40,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:55:40,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:55:41,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:43,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:45,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:55:45,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:47,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:49,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:49,939 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:55:50,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:55:51,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:53,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:55,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:55:55,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:55:55,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:55:55,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:56,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:55:57,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:55:59,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:01,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:01,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:56:03,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:05,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:07,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:56:07,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:09,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:10,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:56:10,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:56:10,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:56:10,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:56:10,925 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:56:10,926 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:56:10,928 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:56:11,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:11,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:56:12,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:12,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:56:13,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:15,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:17,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:18,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:56:19,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:19,941 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:56:21,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:23,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:56:23,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:25,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:56:25,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:56:25,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:56:25,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:27,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:28,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:56:29,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:31,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:33,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:34,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:56:35,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:37,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:39,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:39,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:56:40,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:56:40,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:56:40,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:56:41,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:43,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:44,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:56:45,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:47,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:49,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:49,943 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:56:49,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:56:51,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:53,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:55,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:56:55,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:56:55,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:56:55,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:56:55,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:55,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:56:55,529 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:56:55,530 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:56:55,533 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:56:56,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:56,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:56:57,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:56:59,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:00,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:57:01,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:03,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:05,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:06,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:57:07,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:09,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:10,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:57:10,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:57:10,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:57:11,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:57:11,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:13,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:15,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:16,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:57:17,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:19,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:19,945 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:57:21,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:22,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:57:23,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:25,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:57:25,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:57:25,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:57:25,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:27,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:27,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:57:29,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:31,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:32,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:57:33,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:35,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:37,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:38,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:57:39,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:40,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:57:40,149 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:57:40,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:57:40,152 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:57:40,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:57:40,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:57:40,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:57:40,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:57:41,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:42,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:43,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:43,687 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:57:45,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:47,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:49,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:57:49,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:49,948 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:57:51,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:53,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:54,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:57:55,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:57:55,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:57:55,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:57:55,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:57,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:59,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:57:59,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:58:01,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:03,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:05,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:58:05,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:07,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:09,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:10,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:58:10,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:58:10,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:58:10,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:58:11,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:13,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:15,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:15,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:58:17,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:19,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:19,968 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:58:20,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:58:21,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:23,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:24,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:58:24,785 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:58:24,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:58:24,787 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:58:25,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:58:25,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:58:25,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:58:25,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:25,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:58:26,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:27,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:58:29,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:31,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:33,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:58:33,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:35,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:37,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:39,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:58:39,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:40,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:58:40,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:58:40,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:58:41,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:43,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:44,368 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:58:45,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:47,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:49,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:58:49,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:49,954 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:58:51,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:53,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:55,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:58:55,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:58:55,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:58:55,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:58:55,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:57,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:58:59,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:00,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:59:01,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:03,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:05,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:59:05,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:07,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:09,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:59:09,350 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:59:09,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:59:09,352 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:59:09,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:09,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:59:10,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:59:10,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:59:10,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:59:10,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:11,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:59:11,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:13,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:15,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:16,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:59:17,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:19,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:19,954 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:59:21,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:21,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:59:23,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:25,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:59:25,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:59:25,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:59:25,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:27,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:59:27,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:29,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:31,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:32,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:59:33,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:35,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:37,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:59:37,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:39,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:40,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:59:40,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:59:40,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:59:41,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:43,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:59:43,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:45,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:47,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:48,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:59:49,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:49,957 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 17:59:51,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:53,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:59:53,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:53,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 17:59:53,802 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 17:59:53,803 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 17:59:53,804 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 17:59:54,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:54,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 17:59:55,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 17:59:55,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 17:59:55,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 17:59:55,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:57,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 17:59:59,123 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 17:59:59,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:01,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:03,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:04,512 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:00:05,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:07,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:09,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:09,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:00:10,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:00:10,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:00:10,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:00:11,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:13,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:15,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:00:15,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:17,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:19,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:19,969 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:00:20,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:00:21,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:23,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:25,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:00:25,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:00:25,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:00:25,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:25,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:00:27,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:29,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:31,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:00:31,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:33,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:35,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:36,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:00:37,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:38,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:00:38,411 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:00:38,411 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:00:38,413 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:00:38,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:00:39,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:40,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:00:40,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:00:40,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:00:40,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:41,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:41,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:00:43,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:45,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:47,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:00:47,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:49,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:49,961 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:00:51,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:52,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:00:53,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:55,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:00:55,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:00:55,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:00:55,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:57,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:00:58,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:00:59,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:01,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:03,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:01:03,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:05,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:07,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:08,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:01:09,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:10,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:01:10,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:01:10,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:01:11,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:13,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:14,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:01:15,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:17,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:19,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:01:19,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:19,963 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:01:21,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:23,243 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:01:23,245 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:01:23,246 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:01:23,247 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:01:23,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:23,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:01:24,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:25,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:01:25,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:01:25,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:01:25,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:01:25,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:27,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:29,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:30,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:01:31,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:33,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:35,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:01:35,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:37,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:39,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:40,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:01:40,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:01:40,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:01:41,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:01:41,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:43,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:45,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:46,320 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:01:47,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:49,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:49,965 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:01:51,655 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:01:51,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:53,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:55,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:01:55,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:01:55,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:01:55,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:56,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:01:57,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:01:59,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:01,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:02,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:02:03,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:05,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:07,629 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:02:07,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:02:07,663 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:02:07,663 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:02:07,665 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:02:07,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:07,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:02:08,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:09,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:10,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:02:10,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:02:10,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:02:11,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:12,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:02:13,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:15,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:17,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:18,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:02:19,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:19,981 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:02:21,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:23,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:02:23,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:25,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:02:25,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:02:25,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:02:25,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:27,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:28,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:02:29,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:31,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:33,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:34,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:02:35,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:37,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:39,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:02:39,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:40,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:02:40,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:02:40,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:02:41,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:43,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:44,984 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:02:45,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:47,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:49,969 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:02:49,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:50,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:02:51,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:52,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:02:52,114 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:02:52,114 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:02:52,116 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:02:52,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:02:53,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:54,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:55,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:02:55,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:02:55,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:02:55,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:02:55,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:57,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:02:59,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:00,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:03:01,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:03,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:05,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:06,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:03:07,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:09,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:10,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:03:10,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:03:10,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:03:11,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:03:12,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:14,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:16,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:16,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:03:18,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:19,971 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:03:20,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:22,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:22,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:03:24,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:25,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:03:25,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:03:25,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:03:26,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:27,721 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:03:28,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:30,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:32,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:33,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:03:34,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:36,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:36,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:03:36,672 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:03:36,672 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:03:36,674 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:03:37,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:03:38,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:38,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:03:39,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:40,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:40,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:03:40,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:03:40,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:03:42,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:43,767 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:03:44,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:46,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:48,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:49,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:03:49,974 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:03:50,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:52,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:54,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:54,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:03:55,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:03:55,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:03:55,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:03:56,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:58,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:03:59,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:04:00,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:02,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:04,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:05,097 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:04:06,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:08,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:10,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:10,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:04:10,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:04:10,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:04:10,331 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:04:12,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:14,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:15,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:04:16,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:18,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:19,987 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:04:20,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:20,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:04:21,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:04:21,108 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:04:21,108 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:04:21,109 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:04:22,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:22,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:04:23,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:24,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:25,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:04:25,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:04:25,331 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:04:26,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:26,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:04:28,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:30,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:31,739 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:04:32,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:34,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:36,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:37,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:04:38,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:40,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:40,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:04:40,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:04:40,331 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:04:42,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:42,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:04:44,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:46,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:47,794 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:04:48,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:49,978 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:04:50,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:52,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:53,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:04:54,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:55,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:04:55,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:04:55,331 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:04:56,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:58,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:04:58,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:05:00,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:02,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:03,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:05:04,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:05,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:05:05,704 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:05:05,704 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:05:05,706 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:05:06,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:06,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:05:07,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:08,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:09,235 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:05:10,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:10,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:05:10,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:05:10,331 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:05:12,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:14,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:14,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:05:16,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:18,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:19,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:05:19,980 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:05:20,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:22,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:24,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:25,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:05:25,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:05:25,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:05:25,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:05:26,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:28,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:30,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:30,604 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:05:32,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:34,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:35,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:05:36,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:38,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:40,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:40,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:05:40,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:05:40,332 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:05:41,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:05:42,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:44,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:46,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:46,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:05:48,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:49,983 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:05:50,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:50,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:05:50,306 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:05:50,306 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:05:50,308 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:05:51,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:05:52,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:05:52,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:53,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:54,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:55,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:05:55,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:05:55,332 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:05:56,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:05:57,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:05:58,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:00,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:02,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:02,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:06:04,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:06,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:08,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:06:08,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:10,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:10,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:06:10,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:06:10,332 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:06:12,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:13,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:06:14,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:16,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:18,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:18,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:06:19,994 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:06:20,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:22,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:24,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:06:24,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:25,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:06:25,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:06:25,332 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:06:26,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:28,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:29,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:06:30,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:32,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:34,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:34,812 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:06:34,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:06:34,845 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:06:34,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:06:34,847 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:06:35,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:06:36,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:37,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:38,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:40,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:06:40,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:40,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:06:40,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:06:40,332 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:06:42,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:44,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:45,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:06:46,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:48,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:49,986 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:06:50,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:50,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:06:52,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:54,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:55,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:06:55,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:06:55,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:06:56,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:06:56,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:06:58,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:00,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:01,590 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:07:02,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:04,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:06,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:06,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:07:08,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:10,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:10,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:07:10,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:07:10,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:07:12,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:12,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:07:14,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:16,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:17,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:07:18,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:19,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:07:19,464 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:07:19,464 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:07:19,466 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:07:19,989 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:07:20,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:20,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:07:21,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:22,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:22,984 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:07:24,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:25,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:07:25,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:07:25,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:07:26,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:28,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:28,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:07:30,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:32,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:33,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:07:34,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:36,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:38,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:38,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:07:40,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:07:40,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:40,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:07:40,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:07:42,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:44,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:44,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:07:46,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:48,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:49,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:07:49,991 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:07:50,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:52,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:54,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:55,192 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:07:55,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:07:55,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:07:55,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:07:56,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:07:58,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:00,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:00,555 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:08:02,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:04,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:08:04,162 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:08:04,162 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:08:04,165 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:08:04,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:04,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:08:05,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:05,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:08:06,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:07,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:09,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:10,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:08:10,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:08:10,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:08:11,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:08:11,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:13,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:15,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:16,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:08:17,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:19,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:20,004 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:08:21,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:21,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:08:23,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:25,320 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:08:25,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:08:25,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:08:25,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:27,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:08:27,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:29,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:31,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:32,594 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:08:33,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:35,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:37,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:37,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:08:39,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:40,320 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:08:40,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:08:40,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:08:41,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:43,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:08:43,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:45,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:47,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:48,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:08:48,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:08:48,626 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:08:48,626 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:08:48,628 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:08:49,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:49,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:08:49,995 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:08:50,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:51,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:53,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:53,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:08:55,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:08:55,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:08:55,334 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:08:55,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:57,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:08:59,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:08:59,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:01,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:03,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:04,605 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:09:05,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:07,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:09,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:09,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:09:10,320 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:09:10,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:09:10,334 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:09:11,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:13,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:15,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:09:15,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:17,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:19,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:19,997 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:09:20,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:09:21,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:23,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:25,320 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:09:25,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:09:25,334 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:09:25,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:26,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:09:27,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:29,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:31,369 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:09:31,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:33,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:09:33,183 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:09:33,183 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:09:33,185 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:09:33,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:33,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:09:34,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:35,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:36,706 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:09:37,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:39,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:40,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:09:40,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:09:40,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:09:41,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:42,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:09:43,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:45,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:47,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:09:47,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:49,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:49,999 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:09:51,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:52,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:09:53,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:55,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:09:55,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:09:55,340 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:09:55,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:57,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:09:58,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:09:59,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:01,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:03,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:10:03,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:05,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:07,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:08,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:10:09,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:10,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:10:10,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:10:10,340 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:10:11,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:13,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:14,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:10:15,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:17,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:17,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:10:17,692 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:10:17,693 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:10:17,695 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:10:18,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:18,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:10:19,449 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:10:19,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:20,013 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:10:21,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:23,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:24,789 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:10:25,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:10:25,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:10:25,340 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:10:25,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:27,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:29,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:30,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:10:31,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:33,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:35,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:10:35,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:37,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:39,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:40,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:10:40,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:10:40,340 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:10:40,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:10:41,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:43,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:45,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:46,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:10:47,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:49,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:50,004 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:10:51,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:10:51,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:53,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:55,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:10:55,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:10:55,340 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:10:55,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:56,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:10:57,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:10:59,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:01,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:02,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:11:02,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:11:02,176 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:11:02,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:11:02,178 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:11:02,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:11:03,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:04,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:05,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:07,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:11:07,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:09,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:10,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:11:10,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:11:10,340 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:11:11,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:12,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:11:13,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:15,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:17,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:18,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:11:19,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:20,006 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:11:21,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:23,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:11:23,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:25,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:11:25,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:11:25,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:11:25,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:27,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:28,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:11:29,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:31,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:33,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:34,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:11:35,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:37,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:39,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:11:39,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:40,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:11:40,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:11:40,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:11:41,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:43,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:44,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:11:45,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:46,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:11:46,751 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:11:46,752 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:11:46,753 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:11:47,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:47,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:11:48,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:49,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:50,008 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:11:50,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:11:51,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:53,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:55,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:11:55,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:11:55,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:11:55,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:11:55,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:57,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:11:59,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:00,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:12:01,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:03,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:05,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:06,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:12:07,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:09,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:10,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:12:10,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:12:10,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:12:11,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:12:11,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:13,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:15,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:16,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:12:17,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:19,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:20,021 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:12:21,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:22,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:12:23,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:25,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:12:25,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:12:25,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:12:25,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:27,675 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:12:27,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:29,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:31,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:12:31,270 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:12:31,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:12:31,272 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:12:31,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:31,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:12:32,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:33,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:12:33,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:35,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:37,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:38,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:12:39,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:40,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:12:40,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:12:40,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:12:41,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:43,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:12:43,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:45,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:47,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:49,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:12:49,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:50,012 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:12:51,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:53,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:54,368 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:12:55,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:12:55,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:12:55,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:12:55,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:57,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:12:59,720 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:12:59,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:01,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:03,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:05,085 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:13:05,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:07,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:09,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:10,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:13:10,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:13:10,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:13:10,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:13:11,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:13,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:15,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:15,812 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:13:15,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:13:15,845 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:13:15,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:13:15,847 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:13:16,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:16,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:13:17,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:19,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:20,014 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:13:21,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:13:21,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:23,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:25,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:13:25,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:13:25,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:13:25,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:26,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:13:27,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:29,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:31,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:31,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:13:33,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:35,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:37,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:13:37,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:39,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:40,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:13:40,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:13:40,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:13:41,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:42,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:13:43,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:45,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:47,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:47,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:13:49,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:50,017 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:13:51,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:53,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:13:53,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:55,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:13:55,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:13:55,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:13:55,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:57,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:13:58,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:13:59,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:00,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:14:00,536 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:14:00,536 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:14:00,538 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:14:00,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:14:01,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:02,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:03,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:04,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:14:05,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:07,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:09,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:14:09,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:10,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:14:10,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:14:10,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:14:11,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:13,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:14,740 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:14:15,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:17,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:19,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:20,030 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:14:20,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:14:21,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:23,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:25,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:14:25,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:14:25,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:14:25,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:14:25,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:27,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:29,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:30,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:14:31,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:33,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:35,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:36,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:14:37,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:39,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:40,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:14:40,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:14:40,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:14:41,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:14:41,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:43,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:45,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:14:45,084 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:14:45,084 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:14:45,086 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:14:45,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:45,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:14:46,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:46,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:14:47,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:49,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:50,021 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:14:51,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:52,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:14:53,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:55,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:14:55,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:14:55,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:14:55,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:57,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:14:57,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:14:59,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:01,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:02,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:15:03,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:05,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:07,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:08,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:15:09,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:10,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:15:10,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:15:10,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:15:11,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:13,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:15:13,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:15,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:17,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:18,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:15:19,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:20,023 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:15:21,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:23,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:24,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:15:25,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:15:25,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:15:25,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:15:25,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:27,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:29,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:15:29,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:15:29,658 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:15:29,659 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:15:29,660 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:15:29,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:29,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:15:30,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:31,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:33,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:34,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:15:35,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:37,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:39,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:40,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:15:40,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:15:40,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:15:40,386 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:15:41,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:43,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:45,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:15:45,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:47,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:49,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:50,026 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:15:50,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:15:51,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:53,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:55,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:15:55,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:15:55,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:15:55,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:56,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:15:57,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:15:59,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:01,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:16:01,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:03,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:05,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:07,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:16:07,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:09,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:10,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:16:10,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:16:10,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:16:11,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:12,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:16:13,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:14,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:16:14,314 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:16:14,315 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:16:14,317 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:16:14,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:16:15,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:16,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:17,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:16:17,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:19,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:20,041 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:16:21,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:23,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:16:23,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:25,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:16:25,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:16:25,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:16:25,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:27,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:28,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:16:29,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:31,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:33,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:16:33,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:35,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:37,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:39,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:16:39,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:40,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:16:40,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:16:40,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:16:41,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:43,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:44,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:16:45,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:47,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:49,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:16:49,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:50,030 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:16:51,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:53,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:55,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:16:55,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:16:55,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:16:55,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:16:55,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:57,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:16:58,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:16:58,824 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:16:58,824 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:16:58,826 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:16:58,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:16:59,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:00,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:17:00,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:01,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:03,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:05,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:17:05,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:07,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:09,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:10,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:17:10,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:17:10,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:17:11,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:17:11,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:13,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:15,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:16,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:17:17,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:19,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:20,032 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:17:21,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:22,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:17:23,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:25,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:17:25,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:17:25,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:17:26,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:27,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:17:28,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:30,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:32,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:32,812 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:17:34,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:36,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:38,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:38,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:17:40,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:40,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:17:40,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:17:40,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:17:42,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:43,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:17:43,517 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:17:43,519 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:17:43,520 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:17:43,521 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:17:44,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:44,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:17:45,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:46,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:48,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:48,831 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:17:50,034 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:17:50,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:52,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:54,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:54,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:17:55,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:17:55,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:17:55,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:17:56,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:58,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:17:59,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:18:00,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:02,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:04,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:04,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:18:06,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:08,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:10,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:10,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:18:10,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:18:10,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:18:10,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:18:12,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:14,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:15,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:18:16,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:18,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:20,047 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:18:20,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:20,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:18:22,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:24,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:25,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:18:25,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:18:25,372 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:18:26,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:26,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:18:28,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:18:28,144 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:18:28,145 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:18:28,148 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:18:28,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:28,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:18:29,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:30,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:31,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:18:32,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:34,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:36,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:37,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:18:38,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:40,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:40,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:18:40,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:18:40,372 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:18:42,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:42,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:18:44,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:46,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:47,699 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:18:48,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:50,039 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:18:50,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:52,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:53,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:18:54,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:55,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:18:55,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:18:55,372 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:18:56,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:58,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:18:58,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:19:00,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:02,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:03,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:19:04,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:06,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:08,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:09,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:19:10,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:10,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:19:10,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:19:10,372 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:19:12,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:12,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:19:12,757 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:19:12,757 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:19:12,759 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:19:13,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:19:14,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:14,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:19:15,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:16,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:18,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:19,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:19:20,042 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:19:20,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:22,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:24,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:25,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:19:25,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:19:25,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:19:25,372 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:19:26,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:28,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:30,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:30,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:19:32,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:34,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:35,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:19:36,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:38,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:40,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:19:40,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:40,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:19:40,372 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:19:41,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:19:42,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:44,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:46,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:46,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:19:48,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:50,045 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:19:50,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:51,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:19:52,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:54,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:55,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:19:55,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:19:55,372 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:19:56,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:57,338 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:19:57,369 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:19:57,371 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:19:57,371 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:19:57,373 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:19:57,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:19:58,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:19:59,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:00,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:02,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:02,690 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:20:04,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:06,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:08,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:20:08,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:10,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:20:10,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:20:10,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:20:10,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:12,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:13,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:20:14,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:16,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:18,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:18,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:20:20,047 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:20:20,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:22,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:24,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:20:24,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:25,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:20:25,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:20:25,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:20:26,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:28,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:29,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:20:30,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:32,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:34,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:34,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:20:36,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:38,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:40,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:20:40,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:20:40,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:20:40,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:20:40,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:41,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:20:41,943 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:20:41,944 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:20:41,946 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:20:42,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:42,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:20:43,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:44,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:45,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:20:46,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:48,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:50,049 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:20:50,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:50,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:20:52,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:54,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:55,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:20:55,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:20:55,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:20:56,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:20:56,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:20:58,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:00,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:01,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:21:02,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:04,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:06,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:06,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:21:08,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:10,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:21:10,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:21:10,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:21:10,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:12,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:21:12,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:14,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:16,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:17,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:21:18,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:20,051 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:21:20,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:22,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:22,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:21:24,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:25,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:21:25,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:21:25,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:21:26,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:26,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:21:26,616 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:21:26,616 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:21:26,618 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:21:27,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:21:28,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:21:28,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:30,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:32,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:33,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:21:34,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:36,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:38,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:39,056 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:21:40,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:21:40,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:21:40,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:21:40,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:42,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:44,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:21:44,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:46,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:48,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:49,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:21:50,053 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:21:50,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:52,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:54,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:55,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:21:55,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:21:55,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:21:55,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:21:56,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:21:58,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:00,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:00,588 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:22:02,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:04,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:05,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:22:06,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:08,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:10,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:22:10,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:22:10,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:22:10,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:11,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:22:11,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:22:11,342 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:22:11,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:22:11,344 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:22:11,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:22:12,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:14,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:16,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:16,662 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:22:18,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:20,055 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:22:20,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:21,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:22:22,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:24,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:25,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:22:25,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:22:25,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:22:26,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:27,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:22:28,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:30,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:32,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:32,669 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:22:34,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:36,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:38,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:22:38,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:40,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:22:40,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:22:40,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:22:40,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:42,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:43,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:22:44,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:46,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:48,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:48,720 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:22:50,057 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:22:50,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:52,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:54,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:22:54,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:55,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:22:55,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:22:55,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:22:55,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:22:55,881 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:22:55,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:22:55,884 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:22:56,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:56,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:22:58,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:22:59,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:23:00,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:02,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:04,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:04,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:23:06,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:08,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:10,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:23:10,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:23:10,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:23:10,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:23:10,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:12,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:14,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:15,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:23:16,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:18,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:20,059 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:23:20,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:20,815 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:23:22,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:24,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:25,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:23:25,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:23:25,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:23:26,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:23:26,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:28,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:30,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:31,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:23:32,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:34,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:36,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:36,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:23:38,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:40,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:23:40,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:23:40,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:23:40,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:23:40,489 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:23:40,490 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:23:40,492 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:23:40,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:40,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:23:42,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:23:42,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:44,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:46,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:47,605 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:23:48,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:50,061 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:23:50,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:52,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:52,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:23:54,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:55,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:23:55,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:23:55,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:23:56,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:23:58,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:23:58,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:00,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:02,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:03,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:24:04,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:06,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:08,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:09,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:24:10,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:24:10,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:24:10,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:24:10,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:12,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:14,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:24:14,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:16,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:18,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:19,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:24:20,063 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:24:20,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:22,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:24,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:25,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:24:25,123 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:24:25,124 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:24:25,125 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:24:25,127 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:24:25,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:24:25,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:24:25,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:24:25,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:24:26,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:28,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:30,461 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:24:30,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:32,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:34,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:35,817 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:24:36,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:38,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:40,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:24:40,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:24:40,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:24:40,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:41,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:24:42,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:44,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:46,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:24:46,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:48,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:50,066 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:24:50,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:51,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:24:52,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:54,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:55,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:24:55,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:24:55,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:24:56,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:24:57,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:24:58,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:00,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:02,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:25:02,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:04,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:06,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:07,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:25:08,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:09,741 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:25:09,743 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:25:09,743 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:25:09,745 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:25:10,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:25:10,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:25:10,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:25:10,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:10,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:25:12,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:13,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:25:14,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:16,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:18,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:18,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:25:20,068 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:25:20,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:22,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:24,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:25:24,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:25,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:25:25,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:25:25,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:25:26,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:28,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:29,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:25:29,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:31,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:33,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:34,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:25:35,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:37,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:39,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:40,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:25:40,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:25:40,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:25:40,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:25:41,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:43,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:45,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:25:45,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:47,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:49,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:50,070 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:25:50,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:25:51,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:53,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:54,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:25:54,426 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:25:54,427 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:25:54,429 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:25:54,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:25:55,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:25:55,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:25:55,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:25:55,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:56,192 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:25:56,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:57,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:25:59,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:01,551 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:26:01,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:03,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:05,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:06,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:26:07,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:09,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:10,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:26:10,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:26:10,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:26:11,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:12,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:26:13,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:15,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:17,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:26:17,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:19,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:20,073 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:26:21,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:22,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:26:23,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:25,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:26:25,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:26:25,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:26:25,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:27,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:28,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:26:29,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:31,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:33,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:26:33,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:35,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:37,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:39,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:26:39,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:26:39,061 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:26:39,062 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:26:39,063 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:26:39,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:39,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:26:40,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:26:40,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:26:40,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:26:40,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:41,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:43,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:44,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:26:45,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:47,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:49,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:26:49,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:50,075 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:26:51,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:53,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:55,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:26:55,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:26:55,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:26:55,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:26:55,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:57,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:26:59,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:00,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:27:01,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:03,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:05,729 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:27:05,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:07,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:09,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:10,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:27:10,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:27:10,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:27:11,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:27:11,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:13,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:15,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:16,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:27:17,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:19,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:20,077 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:27:21,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:21,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:27:23,741 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:27:23,742 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:27:23,743 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:27:23,745 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:27:23,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:23,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:27:24,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:25,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:27:25,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:27:25,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:27:25,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:27,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:27:27,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:29,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:31,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:32,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:27:33,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:35,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:37,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:37,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:27:39,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:40,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:27:40,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:27:40,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:27:41,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:43,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:27:43,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:45,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:47,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:48,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:27:49,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:50,080 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:27:51,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:53,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:54,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:27:55,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:27:55,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:27:55,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:27:55,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:58,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:27:59,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:28:00,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:02,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:04,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:04,669 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:28:06,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:08,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:08,256 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:28:08,258 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:28:08,258 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:28:08,260 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:28:09,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:09,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:28:10,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:28:10,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:10,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:28:10,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:28:10,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:28:12,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:14,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:15,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:28:16,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:18,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:20,081 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:28:20,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:20,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:28:22,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:24,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:25,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:28:25,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:28:25,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:28:26,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:28:26,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:28,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:30,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:31,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:28:32,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:34,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:36,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:36,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:28:38,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:40,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:40,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:28:40,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:28:40,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:28:42,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:42,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:28:44,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:46,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:47,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:28:48,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:50,094 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:28:50,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:52,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:52,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:28:53,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:28:53,035 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:28:53,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:28:53,038 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:28:53,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:28:54,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:55,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:55,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:28:55,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:28:55,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:28:56,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:58,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:28:58,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:29:00,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:02,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:03,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:29:04,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:06,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:08,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:09,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:29:10,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:10,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:29:10,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:29:10,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:29:12,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:14,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:14,356 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:29:16,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:18,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:19,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:29:20,086 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:29:20,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:22,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:24,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:25,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:29:25,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:29:25,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:29:25,377 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:29:26,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:28,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:30,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:30,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:29:32,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:34,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:35,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:29:36,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:37,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:29:37,566 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:29:37,567 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:29:37,568 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:29:38,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:38,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:29:39,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:40,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:40,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:29:40,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:29:40,377 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:29:41,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:29:42,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:44,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:46,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:46,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:29:48,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:50,088 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:29:50,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:51,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:29:52,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:54,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:55,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:29:55,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:29:55,409 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:29:56,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:29:57,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:29:58,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:00,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:02,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:02,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:30:04,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:06,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:07,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:30:08,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:10,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:10,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:30:10,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:30:10,390 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:30:12,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:13,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:30:14,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:16,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:18,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:18,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:30:20,090 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:30:20,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:22,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:30:22,134 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:30:22,135 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:30:22,136 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:30:22,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:22,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:30:23,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:23,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:30:24,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:25,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:30:25,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:30:25,390 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:30:26,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:28,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:29,239 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:30:30,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:32,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:34,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:34,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:30:36,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:38,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:39,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:30:40,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:40,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:30:40,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:30:40,390 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:30:42,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:44,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:45,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:30:46,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:48,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:50,104 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:30:50,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:50,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:30:52,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:54,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:55,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:30:55,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:30:55,390 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:30:56,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:30:56,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:30:58,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:00,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:01,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:31:02,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:04,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:06,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:06,746 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:31:06,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:31:06,779 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:31:06,780 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:31:06,781 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:31:07,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:31:08,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:09,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:10,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:10,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:31:10,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:31:10,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:31:12,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:31:12,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:14,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:16,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:17,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:31:18,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:20,095 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:31:20,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:22,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:22,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:31:24,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:25,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:31:25,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:31:25,390 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:31:26,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:28,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:31:28,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:30,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:32,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:33,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:31:34,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:36,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:38,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:38,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:31:40,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:31:40,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:40,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:31:40,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:31:42,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:44,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:31:44,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:46,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:48,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:49,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:31:50,096 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:31:50,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:51,471 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:31:51,472 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:31:51,473 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:31:51,474 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:31:52,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:52,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:31:53,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:54,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:55,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:31:55,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:31:55,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:31:55,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:31:56,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:31:58,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:00,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:32:00,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:02,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:04,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:05,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:32:06,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:08,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:10,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:32:10,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:32:10,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:32:10,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:11,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:32:12,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:14,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:16,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:32:16,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:18,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:20,099 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:32:20,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:21,706 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:32:22,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:24,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:25,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:32:25,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:32:25,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:32:26,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:27,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:32:28,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:30,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:32,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:32:32,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:34,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:36,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:32:36,001 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:32:36,002 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:32:36,004 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:32:36,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:36,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:32:37,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:37,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:32:38,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:40,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:32:40,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:32:40,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:32:40,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:42,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:43,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:32:44,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:46,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:48,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:48,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:32:50,111 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:32:50,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:52,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:53,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:32:54,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:55,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:32:55,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:32:55,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:32:56,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:58,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:32:59,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:33:00,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:02,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:04,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:04,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:33:06,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:08,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:09,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:33:10,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:33:10,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:33:10,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:33:10,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:12,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:14,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:15,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:33:16,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:18,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:20,103 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:33:20,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:20,637 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:33:20,669 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:33:20,671 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:33:20,671 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:33:20,673 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:33:21,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:33:22,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:23,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:24,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:25,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:33:25,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:33:25,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:33:25,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:33:26,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:28,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:30,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:31,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:33:32,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:34,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:36,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:36,720 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:33:38,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:40,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:33:40,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:33:40,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:33:40,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:42,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:33:42,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:44,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:46,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:47,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:33:48,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:50,105 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:33:50,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:52,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:52,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:33:54,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:55,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:33:55,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:33:55,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:33:56,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:33:58,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:33:58,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:00,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:02,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:03,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:34:04,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:05,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:34:05,231 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:34:05,232 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:34:05,234 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:34:05,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:34:06,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:07,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:08,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:08,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:34:10,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:34:10,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:34:10,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:34:10,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:12,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:14,138 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:34:14,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:16,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:18,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:19,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:34:20,107 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:34:20,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:22,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:24,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:24,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:34:25,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:34:25,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:34:25,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:34:26,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:28,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:30,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:34:30,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:32,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:34,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:35,631 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:34:36,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:38,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:40,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:34:40,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:34:40,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:34:40,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:40,975 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:34:42,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:44,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:46,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:34:46,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:48,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:49,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:34:49,904 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:34:49,904 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:34:49,906 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:34:50,129 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:34:50,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:50,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:34:51,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:34:52,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:54,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:55,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:34:55,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:34:55,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:34:56,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:34:56,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:34:58,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:00,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:02,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:35:02,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:04,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:06,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:07,687 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:35:08,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:10,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:35:10,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:35:10,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:35:10,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:12,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:13,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:35:14,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:16,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:18,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:35:18,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:20,112 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:35:20,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:22,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:23,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:35:24,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:25,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:35:25,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:35:25,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:35:26,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:28,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:29,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:35:30,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:32,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:34,422 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:35:34,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:35:34,455 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:35:34,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:35:34,458 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:35:34,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:34,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:35:36,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:38,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:39,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:35:40,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:35:40,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:35:40,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:35:40,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:42,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:44,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:45,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:35:46,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:48,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:50,114 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:35:50,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:35:50,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:52,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:54,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:55,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:35:55,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:35:55,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:35:55,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:35:56,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:35:58,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:00,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:01,141 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:36:02,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:04,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:06,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:36:06,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:08,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:10,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:36:10,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:36:10,393 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:36:10,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:11,536 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:36:12,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:14,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:16,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:17,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:36:18,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:19,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:36:19,006 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:36:19,006 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:36:19,008 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:36:19,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:36:20,116 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:36:20,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:22,537 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:36:22,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:24,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:25,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:36:25,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:36:25,393 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:36:26,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:27,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:36:28,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:30,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:32,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:33,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:36:34,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:36,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:38,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:36:38,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:40,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:36:40,407 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:36:40,407 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:36:40,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:42,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:43,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:36:44,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:46,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:48,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:49,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:36:50,131 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:36:50,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:52,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:54,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:36:54,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:55,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:36:55,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:36:55,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:36:56,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:58,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:36:59,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:37:00,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:02,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:03,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:37:03,581 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:37:03,581 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:37:03,583 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:37:03,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:37:04,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:05,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:37:06,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:08,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:10,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:37:10,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:37:10,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:37:10,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:37:10,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:12,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:14,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:16,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:37:16,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:18,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:20,121 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:37:20,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:21,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:37:22,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:24,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:25,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:37:25,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:37:25,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:37:26,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:37:26,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:28,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:30,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:32,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:37:32,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:34,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:36,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:37,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:37:38,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:40,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:37:40,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:37:40,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:37:40,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:42,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:42,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:37:44,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:46,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:48,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:37:48,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:37:48,197 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:37:48,198 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:37:48,200 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:37:48,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:48,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:37:50,123 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:37:50,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:52,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:53,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:37:54,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:55,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:37:55,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:37:55,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:37:56,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:58,851 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:37:58,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:37:59,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:01,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:03,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:04,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:38:05,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:08,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:09,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:38:10,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:10,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:38:10,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:38:10,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:38:12,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:14,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:14,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:38:16,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:18,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:20,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:20,125 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:38:20,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:38:22,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:24,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:25,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:38:25,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:38:25,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:38:25,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:38:26,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:28,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:30,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:30,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:38:32,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:32,809 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:38:32,811 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:38:32,811 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:38:32,813 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:38:33,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:38:34,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:35,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:36,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:36,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:38:38,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:40,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:40,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:38:40,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:38:40,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:38:41,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:38:42,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:44,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:46,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:47,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:38:48,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:50,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:50,138 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:38:52,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:52,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:38:54,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:55,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:38:55,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:38:55,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:38:56,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:38:57,748 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:38:58,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:00,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:02,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:03,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:39:04,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:06,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:08,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:08,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:39:10,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:10,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:39:10,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:39:10,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:39:12,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:13,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:39:14,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:16,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:17,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:39:17,394 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:39:17,395 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:39:17,398 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:39:18,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:18,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:39:19,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:39:19,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:20,131 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:39:20,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:22,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:24,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:24,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:39:25,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:39:25,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:39:25,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:39:26,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:28,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:29,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:39:30,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:32,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:34,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:35,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:39:36,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:38,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:40,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:40,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:39:40,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:39:40,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:39:40,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:39:42,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:44,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:45,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:39:46,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:48,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:50,133 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:39:50,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:51,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:39:52,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:54,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:55,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:39:55,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:39:55,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:39:56,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:39:56,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:39:58,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:00,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:01,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:40:02,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:40:02,030 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:40:02,031 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:40:02,032 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:40:02,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:02,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:40:03,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:04,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:06,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:07,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:40:08,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:10,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:10,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:40:10,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:40:10,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:40:12,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:12,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:40:14,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:16,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:18,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:40:18,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:20,135 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:40:20,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:22,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:23,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:40:24,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:25,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:40:25,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:40:25,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:40:26,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:28,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:28,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:40:30,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:32,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:34,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:40:34,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:36,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:38,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:39,431 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:40:40,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:40,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:40:40,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:40:40,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:40:42,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:44,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:44,767 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:40:46,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:46,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:40:46,582 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:40:46,582 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:40:46,584 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:40:47,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:40:48,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:49,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:50,111 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:40:50,150 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:40:50,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:52,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:54,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:55,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:40:55,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:40:55,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:40:55,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:40:56,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:40:58,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:00,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:00,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:41:02,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:04,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:06,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:41:06,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:08,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:10,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:10,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:41:10,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:41:10,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:41:11,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:41:12,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:14,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:16,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:16,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:41:18,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:20,139 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:41:20,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:22,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:41:22,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:24,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:25,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:41:25,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:41:25,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:41:26,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:27,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:41:28,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:30,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:31,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:41:31,054 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:41:31,054 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:41:31,056 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:41:31,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:41:32,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:32,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:41:34,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:36,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:38,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:41:38,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:40,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:40,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:41:40,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:41:40,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:41:42,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:43,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:41:44,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:46,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:48,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:48,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:41:50,141 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:41:50,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:52,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:54,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:41:54,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:55,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:41:55,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:41:55,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:41:56,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:58,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:41:59,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:42:00,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:02,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:04,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:04,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:42:06,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:08,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:10,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:42:10,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:10,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:42:10,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:42:10,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:42:12,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:14,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:15,590 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:42:15,622 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:42:15,624 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:42:15,624 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:42:15,626 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:42:16,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:16,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:42:18,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:20,144 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:42:20,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:20,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:42:22,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:24,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:25,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:42:25,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:42:25,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:42:26,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:42:26,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:28,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:30,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:31,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:42:32,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:34,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:36,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:37,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:42:38,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:40,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:40,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:42:40,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:42:40,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:42:42,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:42,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:42:44,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:46,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:47,720 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:42:48,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:50,158 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:42:50,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:52,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:53,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:42:54,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:55,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:42:55,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:42:55,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:42:56,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:58,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:42:58,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:43:00,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:43:00,229 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:43:00,229 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:43:00,231 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:43:00,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:00,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:43:02,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:03,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:43:04,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:06,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:08,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:09,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:43:10,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:10,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:43:10,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:43:10,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:43:12,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:14,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:14,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:43:16,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:18,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:19,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:43:20,148 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:43:20,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:22,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:24,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:25,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:43:25,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:43:25,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:43:25,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:43:26,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:28,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:30,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:30,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:43:32,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:34,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:35,811 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:43:36,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:38,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:40,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:43:40,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:43:40,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:43:40,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:41,143 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:43:42,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:44,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:44,730 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:43:44,732 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:43:44,733 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:43:44,734 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:43:45,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:43:46,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:46,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:43:48,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:50,150 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:43:50,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:51,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:43:52,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:54,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:55,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:43:55,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:43:55,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:43:56,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:43:57,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:43:58,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:00,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:02,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:02,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:44:04,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:06,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:07,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:44:08,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:10,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:44:10,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:44:10,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:44:10,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:12,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:13,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:44:14,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:16,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:18,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:18,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:44:20,152 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:44:20,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:22,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:23,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:44:24,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:25,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:44:25,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:44:25,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:44:26,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:28,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:29,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:44:29,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:44:29,333 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:44:29,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:44:29,334 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:44:29,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:44:30,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:32,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:34,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:34,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:44:36,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:38,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:39,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:44:40,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:44:40,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:44:40,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:44:40,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:42,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:44,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:45,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:44:46,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:48,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:50,154 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:44:50,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:50,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:44:52,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:54,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:55,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:44:55,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:44:55,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:44:55,987 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:44:56,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:44:58,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:00,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:01,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:45:02,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:04,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:06,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:06,730 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:45:08,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:10,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:45:10,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:45:10,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:45:10,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:12,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:45:12,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:13,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:45:13,915 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:45:13,916 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:45:13,917 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:45:14,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:14,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:45:16,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:17,459 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:45:18,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:20,156 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:45:20,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:22,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:22,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:45:24,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:25,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:45:25,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:45:25,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:45:26,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:28,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:45:28,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:30,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:32,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:33,471 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:45:34,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:36,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:38,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:38,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:45:40,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:45:40,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:45:40,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:45:40,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:42,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:44,186 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:45:44,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:46,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:48,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:49,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:45:50,159 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:45:50,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:52,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:54,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:54,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:45:55,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:45:55,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:45:55,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:45:56,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:58,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:45:58,592 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:45:58,594 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:45:58,594 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:45:58,596 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:45:59,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:46:00,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:46:00,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:02,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:04,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:05,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:46:06,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:08,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:10,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:46:10,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:46:10,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:46:10,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:11,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:46:12,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:14,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:16,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:46:16,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:18,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:20,160 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:46:20,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:21,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:46:22,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:24,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:25,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:46:25,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:46:25,416 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:46:26,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:27,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:46:28,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:30,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:32,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:46:32,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:34,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:36,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:37,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:46:38,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:40,407 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:46:40,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:46:40,414 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:46:40,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:42,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:43,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:46:43,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:46:43,180 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:46:43,180 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:46:43,182 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:46:43,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:46:44,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:44,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:46:44,746 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:46:44,746 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:46:44,749 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:46:45,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:46:46,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:48,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:46:50,173 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:46:50,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:46:54,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:46:55,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:46:55,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:46:55,981 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:46:59,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:47:00,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:02,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:04,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:04,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:47:06,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:08,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:10,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:47:10,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:47:10,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:47:10,451 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:47:10,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:12,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:14,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:15,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:47:16,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:18,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:20,165 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:47:20,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:21,166 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:47:22,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:24,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:25,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:47:25,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:47:25,451 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:47:26,597 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:47:26,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:28,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:30,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:31,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:47:32,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:34,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:36,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:37,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:47:38,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:40,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:47:40,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:47:40,451 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:47:40,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:42,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:47:42,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:43,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:47:43,313 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:47:43,313 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:47:43,315 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:47:43,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:47:44,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:46,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:47,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:47:48,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:50,167 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:47:50,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:52,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:53,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:47:54,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:55,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:47:55,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:47:55,451 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:47:56,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:47:58,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:47:58,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:00,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:02,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:03,973 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:48:04,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:06,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:08,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:09,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:48:10,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:48:10,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:48:10,451 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:48:10,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:12,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:14,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:48:14,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:16,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:18,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:20,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:48:20,169 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:48:20,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:22,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:24,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:25,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:48:25,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:48:25,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:48:25,452 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:48:26,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:27,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:48:27,990 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:48:27,990 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:48:27,992 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:48:28,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:28,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:48:30,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:48:30,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:32,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:34,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:36,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:48:36,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:38,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:40,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:48:40,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:48:40,503 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:48:40,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:41,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:48:42,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:44,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:46,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:48:46,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:48,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:50,181 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:48:50,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:52,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:48:52,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:54,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:55,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:48:55,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:48:55,484 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:48:56,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:48:57,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:48:58,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:00,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:02,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:49:02,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:04,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:06,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:08,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:49:08,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:10,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:49:10,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:49:10,484 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:49:10,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:12,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:49:12,610 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:49:12,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:49:12,612 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:49:12,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:12,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:49:13,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:49:14,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:16,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:18,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:49:18,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:20,173 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:49:20,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:22,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:24,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:49:24,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:25,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:49:25,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:49:25,484 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:49:26,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:28,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:29,598 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:49:30,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:32,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:34,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:49:34,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:36,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:38,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:40,338 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:49:40,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:49:40,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:49:40,484 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:49:40,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:42,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:44,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:45,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:49:46,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:49,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:50,176 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:49:51,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:51,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:49:53,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:55,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:55,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:49:55,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:49:55,484 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:49:56,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:49:57,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:49:57,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:49:57,210 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:49:57,211 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:49:57,214 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:49:58,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:49:59,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:01,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:01,734 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:50:03,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:05,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:07,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:07,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:50:09,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:10,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:50:10,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:50:10,484 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:50:11,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:12,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:50:13,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:15,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:17,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:17,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:50:19,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:20,178 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:50:21,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:23,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:23,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:50:25,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:25,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:50:25,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:50:25,484 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:50:27,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:28,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:50:29,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:31,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:33,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:33,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:50:35,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:37,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:39,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:39,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:50:40,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:50:40,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:50:40,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:50:41,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:41,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:50:41,775 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:50:41,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:50:41,777 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:50:42,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:50:43,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:44,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:50:45,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:47,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:49,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:49,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:50:50,190 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:50:51,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:53,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:55,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:55,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:50:55,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:50:55,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:50:55,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:50:57,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:50:59,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:00,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:51:01,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:03,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:05,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:05,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:51:07,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:09,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:10,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:51:10,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:51:10,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:51:11,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:11,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:51:13,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:15,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:17,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:17,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:51:19,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:20,182 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:51:21,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:22,736 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:51:23,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:25,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:25,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:51:25,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:51:25,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:51:26,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:51:26,329 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:51:26,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:51:26,331 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:51:27,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:27,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:51:28,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:51:29,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:31,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:33,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:33,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:51:35,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:37,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:38,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:51:39,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:40,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:51:40,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:51:40,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:51:41,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:43,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:44,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:51:45,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:47,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:49,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:49,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:51:50,185 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:51:51,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:53,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:54,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:51:55,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:55,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:51:55,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:51:55,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:51:57,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:51:59,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:00,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:52:01,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:03,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:05,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:05,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:52:07,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:09,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:10,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:52:10,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:52:10,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:52:10,582 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:52:10,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:52:10,847 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:52:10,847 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:52:10,849 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:52:11,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:11,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:52:13,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:15,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:16,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:52:17,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:19,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:20,186 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:52:21,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:21,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:52:23,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:25,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:25,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:52:25,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:52:25,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:52:26,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:52:27,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:29,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:31,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:32,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:52:33,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:35,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:37,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:37,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:52:39,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:40,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:52:40,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:52:40,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:52:41,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:42,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:52:43,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:45,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:47,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:48,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:52:49,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:50,200 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:52:51,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:53,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:53,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:52:55,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:55,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:52:55,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:52:55,461 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:52:55,461 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:52:55,463 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:52:55,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:52:55,516 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:52:56,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:52:57,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:52:59,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:52:59,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:01,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:03,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:04,355 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:53:05,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:07,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:09,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:09,699 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:53:10,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:53:10,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:53:10,503 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:53:11,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:13,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:15,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:53:15,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:17,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:19,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:20,231 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:53:20,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:53:21,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:23,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:25,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:25,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:53:25,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:53:25,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:53:25,503 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:53:27,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:28,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:30,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:31,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:53:32,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:34,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:36,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:36,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:53:38,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:40,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:53:40,052 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:53:40,052 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:53:40,054 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:53:40,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:40,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:53:40,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:53:40,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:53:40,503 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:53:41,555 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:53:42,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:44,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:46,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:47,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:53:48,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:50,233 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:53:50,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:52,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:52,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:53:54,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:55,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:53:55,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:53:55,503 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:53:56,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:53:57,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:53:58,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:00,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:02,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:03,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:54:04,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:06,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:08,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:08,567 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:54:10,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:10,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:54:10,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:54:10,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:54:12,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:13,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:54:14,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:16,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:18,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:19,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:54:20,235 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:54:20,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:22,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:24,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:24,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:54:24,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:54:24,679 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:54:24,679 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:54:24,681 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:54:25,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:54:25,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:54:25,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:54:25,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:54:26,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:28,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:29,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:54:30,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:32,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:34,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:35,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:54:36,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:38,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:40,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:40,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:54:40,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:54:40,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:54:40,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:54:42,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:44,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:46,085 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:54:46,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:48,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:50,247 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:54:50,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:51,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:54:52,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:54,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:55,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:54:55,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:54:55,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:54:56,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:54:56,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:54:58,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:00,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:02,202 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:55:02,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:04,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:06,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:07,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:55:08,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:09,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:55:09,400 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:55:09,400 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:55:09,402 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:55:10,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:10,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:55:10,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:55:10,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:55:10,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:55:12,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:12,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:55:14,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:16,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:18,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:55:18,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:20,239 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:55:20,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:22,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:23,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:55:24,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:25,501 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:55:25,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:55:25,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:55:26,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:28,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:28,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:55:30,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:32,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:34,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:55:34,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:36,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:38,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:39,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:55:40,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:40,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:55:40,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:55:40,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:55:42,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:44,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:45,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:55:46,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:48,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:50,242 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:55:50,243 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:55:50,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:52,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:54,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:55:54,014 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:55:54,014 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:55:54,016 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:55:54,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:54,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:55:55,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:55:55,501 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:55:55,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:55:55,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:55:56,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:55:58,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:00,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:01,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:56:02,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:04,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:06,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:06,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:56:08,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:10,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:10,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:56:10,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:56:10,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:56:11,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:56:12,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:14,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:16,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:17,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:56:18,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:20,243 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:56:20,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:22,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:22,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:56:24,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:25,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:56:25,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:56:25,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:56:26,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:27,852 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:56:28,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:30,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:32,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:33,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:56:34,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:36,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:38,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:38,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:56:38,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:56:38,654 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:56:38,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:56:38,656 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:56:39,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:56:40,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:40,499 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:56:40,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:56:40,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:56:42,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:43,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:56:44,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:46,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:48,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:49,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:56:50,246 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:56:50,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:52,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:54,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:54,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:56:55,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:56:55,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:56:55,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:56:56,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:56:58,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:00,045 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:57:00,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:02,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:04,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:05,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:57:06,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:08,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:10,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:10,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:57:10,501 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:57:10,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:57:10,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:57:12,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:14,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:16,046 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:57:16,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:18,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:20,248 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:57:20,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:21,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:57:22,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:23,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:57:23,206 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:57:23,206 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:57:23,208 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:57:23,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:57:24,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:25,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:57:25,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:57:25,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:57:26,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:26,622 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:57:28,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:30,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:32,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:57:32,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:34,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:36,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:37,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:57:38,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:40,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:57:40,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:57:40,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:57:40,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:42,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:42,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:57:44,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:46,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:48,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:57:48,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:50,250 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:57:50,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:52,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:53,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:57:54,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:55,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:57:55,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:57:55,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:57:56,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:58,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:57:58,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:58:00,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:02,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:04,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:58:04,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:06,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:07,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:58:07,904 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:58:07,905 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:58:07,907 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:58:08,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:08,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:58:09,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:58:10,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:58:10,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:58:10,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:58:10,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:12,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:14,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:14,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:58:16,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:18,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:20,252 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:58:20,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:58:20,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:22,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:24,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:25,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:58:25,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:58:25,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:58:25,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:58:26,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:28,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:30,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:31,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:58:32,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:34,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:36,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:58:36,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:38,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:40,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:58:40,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:58:40,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:58:40,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:41,631 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:58:42,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:44,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:46,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:47,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:58:48,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:50,255 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:58:50,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:52,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:58:52,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:58:52,486 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:58:52,486 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:58:52,488 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:58:52,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:52,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:58:54,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:55,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:58:55,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:58:55,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:58:56,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:58:57,808 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:58:58,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:00,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:02,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:03,168 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:59:04,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:06,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:08,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:59:08,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:10,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:59:10,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:59:10,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:59:10,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:12,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:13,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:59:14,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:16,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:18,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:19,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:59:20,257 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:59:20,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:22,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:24,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:59:24,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:25,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:59:25,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:59:25,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:59:26,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:28,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:30,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:59:30,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:32,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:34,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:35,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:59:36,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:37,228 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 18:59:37,230 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 18:59:37,230 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 18:59:37,232 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 18:59:37,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 18:59:38,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:40,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:59:40,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:59:40,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:59:40,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:59:40,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:42,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:44,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:46,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:59:46,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:48,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:50,259 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 18:59:50,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:51,260 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:59:52,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:54,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:55,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 18:59:55,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 18:59:55,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 18:59:56,558 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 18:59:56,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 18:59:58,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:00,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:02,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:00:02,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:04,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:06,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:07,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:00:08,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:10,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:00:10,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:00:10,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:00:10,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:12,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:00:12,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:14,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:16,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:18,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:00:18,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:20,261 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:00:20,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:21,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:00:21,781 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:00:21,781 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:00:21,785 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:00:21,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:00:22,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:23,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:00:24,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:25,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:00:25,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:00:25,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:00:26,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:28,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:00:28,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:30,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:32,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:34,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:00:34,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:36,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:38,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:39,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:00:40,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:00:40,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:00:40,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:00:40,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:42,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:44,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:00:44,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:46,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:48,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:50,263 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:00:50,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:00:50,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:52,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:54,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:55,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:00:55,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:00:55,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:00:55,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:00:56,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:00:58,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:00,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:01,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:01:02,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:04,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:06,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:01:06,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:01:06,457 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:01:06,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:01:06,459 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:01:06,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:06,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:01:08,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:10,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:01:10,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:01:10,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:01:10,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:11,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:01:12,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:14,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:16,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:17,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:01:18,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:20,266 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:01:21,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:22,471 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:01:23,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:25,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:25,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:01:25,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:01:25,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:01:27,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:27,823 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:01:29,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:31,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:33,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:33,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:01:35,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:37,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:38,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:01:39,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:40,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:01:40,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:01:40,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:01:41,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:43,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:43,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:01:45,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:47,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:49,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:49,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:01:50,268 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:01:51,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:01:51,008 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:01:51,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:01:51,010 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:01:51,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:51,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:01:53,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:54,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:01:55,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:55,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:01:55,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:01:55,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:01:57,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:59,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:01:59,921 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:02:01,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:03,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:05,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:05,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:02:07,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:09,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:10,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:02:10,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:02:10,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:02:10,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:02:11,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:13,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:15,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:15,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:02:17,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:19,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:20,270 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:02:21,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:21,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:02:23,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:25,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:25,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:02:25,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:02:25,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:02:26,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:02:27,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:29,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:31,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:31,985 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:02:33,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:35,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:35,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:02:35,589 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:02:35,589 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:02:35,591 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:02:36,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:02:37,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:37,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:02:39,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:40,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:02:40,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:02:40,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:02:41,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:42,699 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:02:43,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:45,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:47,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:48,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:02:49,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:50,272 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:02:51,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:53,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:53,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:02:55,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:55,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:02:55,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:02:55,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:02:57,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:02:58,795 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:02:59,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:01,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:03,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:04,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:03:05,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:07,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:09,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:09,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:03:10,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:03:10,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:03:10,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:03:11,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:13,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:14,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:03:15,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:17,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:19,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:20,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:03:20,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:03:20,276 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:03:20,277 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:03:20,277 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:03:20,279 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:03:21,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:21,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:03:23,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:25,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:25,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:03:25,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:03:25,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:03:25,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:03:30,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:03:31,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:33,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:35,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:36,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:03:37,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:39,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:40,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:03:40,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:03:40,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:03:41,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:41,639 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:03:43,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:45,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:46,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:03:47,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:49,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:50,276 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:03:51,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:52,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:03:53,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:55,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:55,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:03:55,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:03:55,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:03:57,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:03:57,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:03:59,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:01,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:03,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:04:03,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:04,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:04:04,989 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:04:04,989 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:04:04,991 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:04:05,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:05,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:04:07,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:08,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:04:09,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:10,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:04:10,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:04:10,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:04:11,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:13,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:13,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:04:15,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:17,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:19,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:19,239 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:04:20,279 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:04:21,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:23,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:24,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:04:25,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:25,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:04:25,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:04:25,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:04:27,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:29,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:29,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:04:31,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:33,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:35,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:35,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:04:37,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:39,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:40,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:04:40,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:04:40,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:04:40,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:04:41,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:43,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:45,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:45,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:04:47,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:49,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:49,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:04:49,507 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:04:49,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:04:49,509 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:04:50,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:04:50,281 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:04:51,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:51,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:04:53,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:55,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:55,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:04:55,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:04:55,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:04:56,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:04:57,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:04:59,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:01,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:02,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:05:03,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:05,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:07,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:07,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:05:09,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:10,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:05:10,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:05:10,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:05:11,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:12,770 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:05:13,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:15,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:17,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:18,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:05:19,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:20,283 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:05:21,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:23,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:23,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:05:25,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:25,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:05:25,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:05:25,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:05:27,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:28,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:05:29,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:31,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:33,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:34,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:05:34,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:05:34,182 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:05:34,183 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:05:34,185 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:05:34,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:05:35,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:37,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:39,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:39,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:05:40,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:05:40,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:05:40,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:05:41,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:43,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:44,818 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:05:45,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:47,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:49,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:50,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:05:50,285 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:05:51,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:53,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:55,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:55,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:05:55,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:05:55,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:05:55,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:05:57,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:05:59,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:00,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:06:01,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:03,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:05,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:06,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:06:07,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:09,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:10,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:06:10,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:06:10,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:06:11,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:11,595 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:06:13,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:15,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:17,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:06:17,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:18,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:06:18,843 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:06:18,843 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:06:18,845 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:06:19,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:19,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:06:20,287 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:06:21,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:22,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:06:23,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:25,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:25,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:06:25,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:06:25,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:06:27,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:27,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:06:29,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:31,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:33,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:06:33,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:35,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:37,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:38,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:06:39,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:40,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:06:40,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:06:40,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:06:41,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:43,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:43,741 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:06:45,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:47,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:49,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:06:49,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:50,289 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:06:51,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:53,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:54,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:06:55,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:55,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:06:55,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:06:55,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:06:57,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:59,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:06:59,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:07:01,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:03,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:03,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:07:03,380 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:07:03,380 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:07:03,382 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:07:04,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:07:05,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:07:05,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:07,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:09,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:10,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:07:10,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:07:10,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:07:10,549 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:07:11,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:13,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:15,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:15,854 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:07:17,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:19,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:20,291 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:07:21,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:07:21,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:23,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:25,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:25,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:07:25,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:07:25,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:07:26,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:07:27,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:29,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:31,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:31,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:07:33,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:35,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:37,231 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:07:37,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:39,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:40,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:07:40,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:07:40,510 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:07:41,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:42,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:07:43,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:45,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:47,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:48,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:07:48,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:07:48,038 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:07:48,038 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:07:48,040 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:07:48,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:07:49,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:50,293 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:07:51,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:53,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:07:53,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:55,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:55,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:07:55,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:07:55,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:07:57,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:07:58,720 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:07:59,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:01,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:03,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:04,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:08:05,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:07,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:09,459 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:08:09,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:10,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:08:10,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:08:10,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:08:10,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:12,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:14,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:14,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:08:16,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:18,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:20,186 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:08:20,296 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:08:20,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:22,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:24,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:25,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:08:25,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:08:25,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:08:25,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:08:26,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:28,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:30,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:30,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:08:32,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:32,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:08:32,714 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:08:32,714 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:08:32,716 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:08:33,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:08:34,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:36,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:08:36,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:38,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:40,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:08:40,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:08:40,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:08:40,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:41,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:08:42,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:44,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:46,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:46,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:08:48,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:50,297 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:08:50,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:52,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:08:52,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:54,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:55,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:08:55,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:08:55,510 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:08:56,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:08:57,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:08:58,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:00,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:02,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:02,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:09:04,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:06,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:08,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:09:08,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:10,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:09:10,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:09:10,510 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:09:10,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:12,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:13,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:09:14,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:16,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:17,262 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:09:17,274 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:09:17,275 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:09:17,276 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:09:17,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:09:18,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:19,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:09:20,299 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:09:20,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:22,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:24,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:09:24,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:25,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:09:25,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:09:25,510 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:09:26,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:28,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:29,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:09:30,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:32,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:34,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:35,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:09:36,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:38,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:40,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:09:40,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:09:40,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:09:40,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:09:40,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:42,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:44,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:45,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:09:46,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:48,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:50,301 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:09:50,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:51,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:09:52,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:54,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:55,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:09:55,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:09:55,526 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:09:56,511 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:09:56,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:09:58,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:00,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:01,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:10:01,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:10:01,876 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:10:01,877 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:10:01,878 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:10:02,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:02,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:10:04,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:06,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:07,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:10:08,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:10,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:10:10,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:10:10,526 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:10:10,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:12,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:10:12,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:14,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:16,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:17,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:10:18,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:20,304 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:10:20,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:22,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:23,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:10:24,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:25,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:10:25,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:10:25,525 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:10:26,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:28,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:10:28,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:30,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:32,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:34,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:10:34,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:36,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:38,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:39,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:10:40,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:10:40,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:10:40,525 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:10:40,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:42,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:44,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:10:44,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:46,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:10:46,567 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:10:46,567 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:10:46,570 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:10:46,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:46,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:10:48,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:50,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:10:50,306 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:10:50,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:52,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:54,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:55,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:10:55,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:10:55,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:10:55,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:10:56,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:10:58,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:00,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:11:00,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:02,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:04,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:06,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:11:06,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:08,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:10,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:11:10,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:11:10,539 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:11:10,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:11,551 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:11:12,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:14,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:16,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:11:16,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:18,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:20,308 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:11:20,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:22,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:11:22,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:24,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:25,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:11:25,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:11:25,539 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:11:26,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:27,569 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:11:28,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:30,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:31,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:11:31,155 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:11:31,156 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:11:31,158 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:11:31,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:11:32,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:11:32,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:34,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:36,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:38,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:11:38,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:40,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:11:40,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:11:40,539 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:11:40,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:42,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:43,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:11:44,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:46,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:48,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:11:48,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:50,310 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:11:50,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:52,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:54,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:11:54,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:55,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:11:55,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:11:55,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:11:56,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:58,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:11:59,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:12:00,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:02,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:04,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:04,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:12:06,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:08,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:10,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:12:10,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:12:10,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:12:10,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:12:10,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:12,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:14,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:15,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:12:15,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:12:15,733 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:12:15,734 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:12:15,735 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:12:15,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:12:16,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:18,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:20,313 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:12:20,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:21,045 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:12:22,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:24,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:25,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:12:25,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:12:25,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:12:26,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:12:26,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:28,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:30,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:31,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:12:32,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:34,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:36,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:37,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:12:38,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:40,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:12:40,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:12:40,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:12:40,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:42,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:12:42,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:44,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:46,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:47,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:12:48,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:50,314 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:12:50,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:52,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:53,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:12:55,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:55,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:12:55,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:12:55,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:12:57,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:12:58,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:12:59,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:00,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:13:00,273 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:13:00,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:13:00,275 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:13:01,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:01,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:13:03,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:03,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:13:05,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:07,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:09,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:09,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:13:10,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:13:10,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:13:10,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:13:11,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:13,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:14,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:13:15,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:17,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:19,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:19,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:13:20,317 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:13:21,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:23,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:25,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:25,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:13:25,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:13:25,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:13:25,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:13:27,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:29,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:30,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:13:31,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:33,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:35,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:35,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:13:37,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:39,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:40,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:13:40,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:13:40,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:13:41,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:41,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:13:43,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:44,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:13:44,800 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:13:44,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:13:44,802 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:13:45,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:45,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:13:46,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:13:47,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:49,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:50,319 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:13:51,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:51,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:13:53,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:55,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:55,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:13:55,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:13:55,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:13:57,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:13:57,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:13:59,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:01,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:02,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:14:03,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:05,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:07,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:07,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:14:09,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:10,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:14:10,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:14:10,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:14:11,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:13,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:13,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:14:15,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:17,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:18,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:14:19,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:20,321 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:14:21,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:23,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:24,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:14:25,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:25,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:14:25,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:14:25,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:14:27,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:29,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:29,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:14:29,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:14:29,412 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:14:29,412 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:14:29,414 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:14:30,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:14:31,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:33,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:34,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:14:35,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:37,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:39,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:40,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:14:40,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:14:40,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:14:40,543 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:14:41,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:43,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:45,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:45,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:14:47,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:49,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:50,323 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:14:50,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:14:51,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:53,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:55,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:55,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:14:55,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:14:55,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:14:56,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:14:57,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:14:59,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:01,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:01,466 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:15:03,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:05,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:06,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:15:07,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:09,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:10,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:15:10,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:15:10,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:15:11,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:12,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:15:13,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:14,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:15:14,014 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:15:14,015 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:15:14,017 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:15:14,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:15:15,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:17,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:17,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:15:19,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:20,325 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:15:21,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:22,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:15:23,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:25,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:25,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:15:25,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:15:25,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:15:27,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:28,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:15:29,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:31,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:33,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:33,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:15:35,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:37,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:38,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:15:39,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:40,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:15:40,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:15:40,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:15:41,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:43,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:44,263 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:15:45,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:47,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:49,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:49,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:15:50,327 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:15:51,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:53,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:54,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:15:55,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:55,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:15:55,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:15:55,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:15:57,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:58,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:15:58,563 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:15:58,563 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:15:58,565 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:15:59,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:15:59,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:16:00,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:16:01,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:03,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:05,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:05,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:16:07,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:09,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:10,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:16:10,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:16:10,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:16:10,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:16:11,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:13,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:15,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:16,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:16:17,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:19,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:20,329 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:16:21,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:21,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:16:23,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:25,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:25,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:16:25,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:16:25,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:16:26,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:16:27,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:29,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:31,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:32,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:16:33,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:35,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:37,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:37,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:16:39,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:40,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:16:40,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:16:40,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:16:41,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:43,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:16:43,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:16:43,094 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:16:43,095 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:16:43,097 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:16:43,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:43,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:16:45,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:47,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:48,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:16:49,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:50,332 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:16:51,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:53,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:53,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:16:55,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:55,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:16:55,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:16:55,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:16:57,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:16:59,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:16:59,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:01,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:03,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:04,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:17:05,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:07,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:09,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:09,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:17:10,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:17:10,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:17:10,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:17:11,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:13,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:15,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:17:15,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:17,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:19,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:20,411 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:17:20,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:17:21,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:23,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:25,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:25,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:17:25,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:17:25,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:17:25,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:17:27,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:27,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:17:27,702 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:17:27,702 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:17:27,704 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:17:28,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:17:29,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:31,238 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:17:31,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:33,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:35,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:36,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:17:37,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:39,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:40,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:17:40,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:17:40,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:17:41,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:41,637 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:17:43,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:45,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:47,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:17:47,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:49,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:50,339 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:17:51,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:52,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:17:53,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:55,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:55,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:17:55,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:17:55,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:17:57,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:17:57,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:17:59,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:01,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:03,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:03,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:18:05,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:07,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:08,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:18:09,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:10,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:18:10,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:18:10,582 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:18:11,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:12,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:18:12,327 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:18:12,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:18:12,329 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:18:12,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:18:13,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:14,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:18:15,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:17,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:19,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:18:19,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:20,341 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:18:21,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:23,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:24,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:18:25,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:25,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:18:25,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:18:25,579 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:18:27,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:29,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:30,111 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:18:31,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:33,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:35,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:18:35,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:37,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:39,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:40,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:18:40,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:18:40,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:18:40,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:18:41,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:43,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:45,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:46,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:18:47,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:49,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:50,343 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:18:51,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:18:51,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:53,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:55,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:18:55,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:18:55,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:18:55,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:56,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:18:56,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:18:56,935 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:18:56,936 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:18:56,937 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:18:57,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:18:57,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:18:59,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:01,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:02,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:19:03,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:05,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:07,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:19:07,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:09,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:10,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:19:10,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:19:10,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:19:11,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:12,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:19:13,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:15,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:17,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:18,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:19:19,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:20,345 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:19:21,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:23,629 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:19:23,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:25,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:19:25,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:19:25,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:19:25,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:27,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:28,985 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:19:29,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:31,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:33,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:34,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:19:35,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:37,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:39,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:39,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:19:40,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:19:40,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:19:40,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:19:41,517 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:19:41,519 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:19:41,519 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:19:41,521 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:19:41,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:41,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:19:43,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:45,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:19:45,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:47,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:49,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:50,347 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:19:50,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:19:51,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:53,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:55,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:19:55,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:19:55,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:19:55,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:19:55,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:57,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:19:59,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:01,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:20:01,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:03,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:05,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:06,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:20:07,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:09,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:10,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:20:10,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:20:10,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:20:11,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:20:11,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:13,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:15,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:17,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:20:17,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:19,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:20,350 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:20:21,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:22,501 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:20:23,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:25,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:20:25,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:20:25,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:20:25,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:26,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:20:26,095 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:20:26,095 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:20:26,097 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:20:26,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:20:27,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:27,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:20:29,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:31,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:33,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:20:33,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:34,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:36,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:38,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:20:38,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:40,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:20:40,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:20:40,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:20:40,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:42,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:43,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:20:44,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:46,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:48,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:49,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:20:50,351 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:20:50,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:52,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:54,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:20:54,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:55,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:20:55,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:20:55,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:20:56,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:58,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:20:59,865 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:21:00,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:02,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:04,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:05,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:21:06,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:08,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:10,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:21:10,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:21:10,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:21:10,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:21:10,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:21:10,657 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:21:10,658 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:21:10,661 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:21:10,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:10,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:21:12,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:14,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:15,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:21:16,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:18,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:20,364 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:21:20,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:21,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:21:22,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:24,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:25,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:21:25,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:21:25,581 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:21:26,635 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:21:26,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:28,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:30,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:31,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:21:32,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:34,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:36,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:37,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:21:38,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:40,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:21:40,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:21:40,581 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:21:40,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:42,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:21:42,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:44,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:46,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:48,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:21:48,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:50,355 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:21:50,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:52,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:53,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:21:54,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:55,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:21:55,193 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:21:55,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:21:55,195 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:21:55,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:21:55,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:21:55,581 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:21:55,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:21:56,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:21:58,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:21:58,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:00,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:02,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:04,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:22:04,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:06,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:08,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:09,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:22:10,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:22:10,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:22:10,581 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:22:10,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:12,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:14,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:22:14,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:16,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:18,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:20,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:22:20,357 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:22:20,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:22,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:24,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:25,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:22:25,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:22:25,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:22:25,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:22:26,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:28,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:30,865 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:22:30,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:32,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:34,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:36,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:22:36,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:38,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:39,819 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:22:39,821 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:22:39,821 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:22:39,823 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:22:39,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:22:40,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:22:40,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:22:40,583 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:22:40,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:41,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:22:42,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:44,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:46,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:46,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:22:48,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:50,360 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:22:50,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:52,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:22:52,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:54,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:55,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:22:55,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:22:55,583 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:22:56,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:22:57,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:22:58,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:00,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:02,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:03,015 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:23:04,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:06,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:08,363 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:23:08,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:10,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:23:10,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:23:10,583 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:23:10,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:12,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:13,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:23:14,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:16,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:18,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:19,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:23:20,373 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:23:20,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:22,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:24,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:23:24,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:23:24,435 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:23:24,435 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:23:24,437 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:23:24,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:24,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:23:25,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:23:25,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:23:25,583 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:23:26,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:28,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:29,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:23:30,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:32,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:34,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:35,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:23:36,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:38,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:40,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:23:40,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:23:40,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:23:40,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:23:40,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:42,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:44,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:45,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:23:46,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:48,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:50,364 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:23:50,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:51,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:23:52,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:54,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:55,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:23:55,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:23:55,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:23:56,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:23:56,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:23:58,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:00,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:01,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:24:02,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:04,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:06,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:07,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:24:08,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:09,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:24:09,057 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:24:09,057 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:24:09,059 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:24:09,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:24:10,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:24:10,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:24:10,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:24:10,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:12,590 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:24:12,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:14,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:16,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:17,931 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:24:18,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:20,366 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:24:20,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:22,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:23,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:24:25,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:25,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:24:25,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:24:25,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:24:27,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:28,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:24:29,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:31,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:33,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:34,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:24:35,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:37,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:39,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:39,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:24:40,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:24:40,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:24:40,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:24:41,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:43,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:44,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:24:45,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:47,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:49,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:50,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:24:50,368 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:24:51,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:53,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:53,740 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:24:53,742 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:24:53,742 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:24:53,744 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:24:54,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:24:55,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:55,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:24:55,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:24:55,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:24:55,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:24:57,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:24:59,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:00,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:25:01,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:03,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:05,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:06,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:25:07,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:09,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:10,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:25:10,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:25:10,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:25:11,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:11,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:25:13,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:15,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:16,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:25:17,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:19,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:20,381 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:25:21,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:22,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:25:23,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:25,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:25,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:25:25,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:25:25,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:25:27,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:27,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:25:29,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:31,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:32,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:25:33,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:35,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:37,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:38,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:25:38,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:25:38,292 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:25:38,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:25:38,294 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:25:39,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:39,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:25:40,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:25:40,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:25:40,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:25:41,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:43,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:43,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:25:45,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:47,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:48,984 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:25:49,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:50,373 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:25:51,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:53,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:54,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:25:55,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:55,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:25:55,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:25:55,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:25:57,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:59,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:25:59,662 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:26:01,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:03,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:05,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:26:05,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:07,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:09,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:10,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:26:10,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:26:10,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:26:10,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:26:11,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:13,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:15,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:15,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:26:17,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:19,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:20,375 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:26:21,047 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:26:21,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:22,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:26:22,861 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:26:22,861 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:26:22,863 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:26:23,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:23,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:26:25,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:25,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:26:25,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:26:25,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:26:26,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:26:27,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:29,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:31,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:31,725 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:26:33,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:35,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:37,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:26:37,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:39,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:40,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:26:40,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:26:40,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:26:41,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:42,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:26:43,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:45,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:47,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:47,746 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:26:49,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:50,377 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:26:51,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:53,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:26:53,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:55,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:55,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:26:55,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:26:55,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:26:57,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:26:58,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:26:59,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:01,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:03,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:03,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:27:05,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:07,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:07,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:27:07,325 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:27:07,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:27:07,327 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:27:08,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:27:09,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:27:09,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:10,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:27:10,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:27:10,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:27:11,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:13,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:14,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:27:15,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:17,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:19,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:19,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:27:20,391 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:27:21,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:23,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:25,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:27:25,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:25,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:27:25,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:27:25,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:27:27,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:29,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:30,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:27:31,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:33,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:35,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:35,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:27:37,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:39,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:40,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:27:40,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:27:40,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:27:41,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:27:41,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:43,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:45,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:46,538 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:27:47,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:49,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:50,381 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:27:51,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:51,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:27:51,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:27:51,929 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:27:51,929 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:27:51,931 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:27:52,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:27:53,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:55,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:55,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:27:55,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:27:55,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:27:57,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:27:57,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:27:59,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:01,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:02,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:28:03,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:05,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:07,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:07,975 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:28:09,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:10,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:28:10,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:28:10,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:28:11,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:13,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:13,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:28:15,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:17,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:18,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:28:19,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:20,383 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:28:21,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:23,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:24,047 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:28:25,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:25,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:28:25,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:28:25,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:28:27,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:29,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:28:29,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:31,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:33,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:34,758 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:28:35,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:36,568 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:28:36,569 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:28:36,570 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:28:36,571 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:28:37,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:37,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:28:39,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:40,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:28:40,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:28:40,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:28:40,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:28:41,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:43,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:45,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:28:45,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:47,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:49,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:50,386 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:28:50,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:28:51,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:53,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:55,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:55,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:28:55,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:28:55,586 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:28:56,166 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:28:57,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:28:59,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:01,512 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:29:01,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:03,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:05,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:06,852 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:29:07,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:09,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:10,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:29:10,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:29:10,618 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:29:11,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:12,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:29:13,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:15,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:17,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:17,663 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:29:19,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:20,405 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:29:21,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:29:21,256 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:29:21,256 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:29:21,258 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:29:21,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:21,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:29:23,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:29:23,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:25,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:25,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:29:25,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:29:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:29:27,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:28,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:29:29,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:31,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:33,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:33,683 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:29:35,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:37,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:39,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:29:39,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:40,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:29:40,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:29:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:29:41,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:43,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:44,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:29:45,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:47,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:49,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:49,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:29:50,390 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:29:51,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:53,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:55,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:29:55,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:55,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:29:55,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:29:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:29:57,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:29:59,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:00,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:30:01,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:03,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:05,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:05,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:30:05,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:30:05,860 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:30:05,861 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:30:05,862 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:30:06,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:30:07,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:09,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:10,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:30:10,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:30:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:30:11,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:30:11,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:13,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:15,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:16,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:30:17,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:19,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:20,392 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:30:21,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:21,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:30:23,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:25,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:30:25,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:30:25,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:30:27,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:30:27,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:29,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:31,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:32,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:30:33,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:35,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:37,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:37,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:30:39,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:40,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:30:40,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:30:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:30:41,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:43,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:30:43,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:45,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:47,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:48,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:30:49,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:50,394 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:30:50,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:30:50,445 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:30:50,445 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:30:50,447 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:30:50,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:30:51,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:53,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:53,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:30:55,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:30:55,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:30:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:30:55,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:57,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:30:59,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:30:59,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:01,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:03,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:04,688 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:31:05,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:07,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:09,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:10,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:31:10,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:31:10,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:31:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:31:11,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:13,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:15,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:31:15,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:17,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:19,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:20,409 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:31:20,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:31:21,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:23,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:25,597 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:31:25,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:31:25,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:31:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:31:25,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:27,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:29,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:30,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:31:31,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:33,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:35,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:31:35,072 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:31:35,072 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:31:35,075 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:31:35,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:35,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:31:36,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:31:37,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:39,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:40,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:31:40,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:31:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:31:41,699 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:31:41,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:43,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:45,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:46,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:31:47,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:49,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:50,399 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:31:51,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:52,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:31:53,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:55,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:31:55,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:31:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:31:55,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:57,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:31:57,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:31:59,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:01,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:02,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:32:03,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:05,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:07,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:08,097 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:32:09,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:10,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:32:10,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:32:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:32:11,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:13,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:32:13,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:15,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:17,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:18,805 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:32:19,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:32:19,623 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:32:19,623 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:32:19,625 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:32:19,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:19,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:32:20,401 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:32:21,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:23,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:24,156 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:32:25,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:32:25,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:32:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:32:25,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:27,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:29,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:32:29,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:31,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:33,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:34,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:32:35,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:37,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:39,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:40,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:32:40,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:32:40,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:32:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:32:41,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:43,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:45,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:32:45,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:47,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:49,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:50,403 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:32:51,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:32:51,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:53,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:55,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:32:55,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:32:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:32:55,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:56,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:32:57,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:32:59,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:01,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:02,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:33:03,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:04,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:33:04,116 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:33:04,117 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:33:04,119 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:33:04,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:33:05,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:07,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:33:07,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:09,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:10,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:33:10,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:33:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:33:11,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:12,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:33:13,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:15,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:17,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:18,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:33:19,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:20,419 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:33:21,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:23,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:33:23,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:25,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:33:25,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:33:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:33:25,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:27,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:28,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:33:29,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:31,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:33,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:34,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:33:35,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:37,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:39,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:33:39,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:40,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:33:40,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:33:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:33:41,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:43,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:45,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:33:45,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:47,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:48,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:33:48,682 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:33:48,682 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:33:48,684 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:33:48,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:33:49,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:50,407 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:33:50,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:33:51,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:53,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:55,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:33:55,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:33:55,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:33:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:33:55,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:57,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:33:59,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:01,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:34:01,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:03,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:05,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:06,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:34:07,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:09,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:10,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:34:10,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:34:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:34:11,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:34:11,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:13,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:15,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:17,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:34:17,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:19,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:20,409 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:34:21,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:22,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:34:23,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:25,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:34:25,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:34:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:34:25,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:27,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:27,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:34:29,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:31,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:33,238 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:34:33,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:34:33,272 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:34:33,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:34:33,274 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:34:33,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:33,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:34:35,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:37,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:38,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:34:39,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:40,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:34:40,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:34:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:34:41,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:43,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:43,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:34:45,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:47,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:49,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:34:49,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:50,411 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:34:51,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:53,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:54,675 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:34:55,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:34:55,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:34:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:34:55,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:57,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:34:59,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:00,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:35:01,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:03,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:05,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:35:05,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:07,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:09,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:10,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:35:10,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:35:10,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:35:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:35:11,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:13,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:15,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:16,109 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:35:17,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:35:17,924 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:35:17,924 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:35:17,926 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:35:17,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:17,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:35:19,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:20,425 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:35:21,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:35:21,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:23,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:25,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:35:25,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:35:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:35:25,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:26,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:35:27,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:29,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:31,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:32,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:35:33,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:35,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:37,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:35:37,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:39,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:40,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:35:40,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:35:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:35:41,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:42,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:35:43,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:45,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:47,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:48,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:35:49,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:50,415 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:35:51,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:53,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:35:53,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:55,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:35:55,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:35:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:35:55,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:57,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:35:58,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:35:59,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:01,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:02,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:36:02,478 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:36:02,479 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:36:02,481 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:36:02,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:36:03,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:04,238 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:36:05,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:07,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:09,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:36:09,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:10,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:36:10,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:36:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:36:11,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:13,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:14,977 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:36:15,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:18,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:20,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:20,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:36:20,417 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:36:22,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:24,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:25,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:36:25,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:36:25,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:36:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:36:26,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:28,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:30,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:31,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:36:32,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:34,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:36,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:36,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:36:38,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:40,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:40,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:36:40,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:36:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:36:41,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:36:42,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:44,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:46,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:47,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:36:47,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:36:47,206 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:36:47,207 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:36:47,209 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:36:48,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:48,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:36:50,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:50,420 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:36:52,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:52,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:36:54,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:55,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:36:55,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:36:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:36:56,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:36:57,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:36:58,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:00,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:02,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:03,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:37:04,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:06,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:08,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:08,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:37:10,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:10,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:37:10,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:37:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:37:12,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:13,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:37:14,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:16,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:18,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:19,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:37:20,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:20,434 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:37:22,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:24,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:24,658 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:37:25,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:37:25,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:37:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:37:26,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:28,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:30,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:37:30,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:31,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:37:31,858 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:37:31,859 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:37:31,860 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:37:32,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:32,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:37:33,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:35,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:37:36,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:38,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:40,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:40,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:37:40,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:37:40,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:37:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:37:42,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:44,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:46,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:46,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:37:48,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:50,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:50,425 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:37:51,427 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:37:52,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:54,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:55,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:55,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:37:55,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:37:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:37:56,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:37:57,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:37:59,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:01,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:02,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:38:03,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:05,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:07,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:07,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:38:09,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:10,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:38:10,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:38:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:38:11,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:12,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:38:13,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:15,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:16,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:38:16,482 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:38:16,483 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:38:16,485 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:38:17,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:17,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:38:18,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:38:19,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:20,427 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:38:21,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:23,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:23,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:38:25,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:25,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:38:25,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:38:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:38:27,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:28,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:38:29,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:31,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:33,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:34,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:38:35,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:37,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:39,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:39,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:38:40,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:38:40,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:38:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:38:41,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:43,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:45,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:38:45,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:47,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:49,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:50,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:38:50,429 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:38:51,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:53,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:55,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:55,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:38:55,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:38:55,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:38:55,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:38:57,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:38:59,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:01,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:39:01,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:39:01,076 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:39:01,076 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:39:01,078 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:39:01,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:01,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:39:03,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:05,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:06,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:39:07,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:09,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:10,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:39:10,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:39:10,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:39:11,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:11,726 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:39:13,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:15,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:17,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:39:17,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:19,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:20,447 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:39:21,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:22,449 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:39:23,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:25,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:25,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:39:25,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:39:25,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:39:27,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:27,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:39:29,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:31,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:33,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:39:33,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:35,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:37,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:38,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:39:39,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:40,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:39:40,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:39:40,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:39:41,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:43,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:43,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:39:45,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:45,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:39:45,710 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:39:45,710 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:39:45,712 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:39:46,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:39:47,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:49,231 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:39:49,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:50,433 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:39:51,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:53,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:54,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:39:55,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:55,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:39:55,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:39:55,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:39:57,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:59,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:39:59,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:40:01,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:03,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:05,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:40:05,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:07,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:09,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:10,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:40:10,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:40:10,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:40:10,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:40:11,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:13,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:15,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:16,046 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:40:17,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:19,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:20,436 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:40:21,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:40:21,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:23,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:25,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:25,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:40:25,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:40:25,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:40:26,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:40:27,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:29,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:30,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:40:30,348 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:40:30,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:40:30,350 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:40:30,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:40:31,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:32,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:40:33,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:35,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:37,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:40:37,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:39,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:40,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:40:40,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:40:40,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:40:41,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:42,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:40:43,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:45,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:47,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:48,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:40:49,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:50,438 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:40:51,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:53,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:40:53,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:55,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:55,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:40:55,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:40:55,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:40:57,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:40:58,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:40:59,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:01,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:03,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:04,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:41:05,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:07,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:09,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:09,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:41:10,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:41:10,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:41:10,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:41:11,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:13,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:14,921 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:41:14,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:41:14,956 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:41:14,957 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:41:14,958 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:41:15,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:15,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:41:17,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:19,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:20,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:41:20,450 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:41:21,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:23,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:25,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:25,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:41:25,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:41:25,615 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:41:25,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:41:27,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:29,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:31,014 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:41:31,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:33,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:35,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:36,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:41:37,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:39,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:40,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:41:40,615 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:41:40,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:41:41,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:41,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:41:43,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:45,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:47,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:41:47,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:49,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:50,442 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:41:51,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:52,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:41:53,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:55,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:55,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:41:55,615 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:41:55,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:41:57,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:57,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:41:59,538 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:41:59,540 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:41:59,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:41:59,543 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:41:59,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:41:59,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:42:01,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:03,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:42:03,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:05,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:07,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:08,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:42:09,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:10,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:42:10,615 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:42:10,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:42:11,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:13,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:13,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:42:15,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:17,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:19,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:42:19,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:20,444 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:42:21,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:23,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:24,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:42:25,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:42:25,615 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:42:25,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:25,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:42:27,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:29,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:29,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:42:31,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:33,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:35,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:42:35,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:37,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:39,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:40,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:42:40,662 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:42:40,663 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:42:40,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:42:41,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:43,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:44,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:42:44,179 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:42:44,180 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:42:44,182 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:42:44,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:42:45,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:45,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:42:47,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:49,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:50,446 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:42:51,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:42:51,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:53,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:55,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:42:55,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:42:55,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:55,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:42:56,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:42:57,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:42:59,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:01,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:02,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:43:03,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:05,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:07,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:43:07,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:09,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:10,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:43:10,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:43:10,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:43:11,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:12,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:43:13,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:15,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:17,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:18,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:43:19,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:20,459 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:43:21,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:23,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:43:23,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:25,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:43:25,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:43:25,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:43:25,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:27,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:28,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:43:28,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:43:28,785 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:43:28,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:43:28,787 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:43:29,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:29,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:43:31,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:33,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:34,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:43:35,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:37,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:39,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:43:39,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:40,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:43:40,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:43:40,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:43:41,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:43,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:44,807 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:43:45,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:47,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:49,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:50,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:43:50,450 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:43:51,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:53,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:55,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:43:55,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:43:55,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:43:55,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:43:55,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:57,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:43:59,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:00,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:44:01,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:03,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:05,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:06,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:44:07,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:09,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:10,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:44:10,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:44:10,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:44:11,592 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:44:11,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:13,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:44:13,402 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:44:13,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:44:13,404 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:44:13,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:13,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:44:15,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:16,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:44:17,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:19,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:20,453 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:44:21,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:22,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:44:23,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:25,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:44:25,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:44:25,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:44:25,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:27,655 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:44:27,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:29,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:31,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:33,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:44:33,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:35,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:37,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:38,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:44:39,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:40,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:44:40,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:44:40,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:44:41,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:43,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:44:43,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:45,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:47,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:49,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:44:49,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:50,455 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:44:51,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:53,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:54,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:44:55,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:44:55,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:44:55,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:44:55,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:57,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:44:58,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:44:58,037 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:44:58,038 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:44:58,039 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:44:58,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:44:59,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:44:59,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:01,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:03,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:05,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:45:05,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:07,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:09,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:10,469 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:45:10,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:45:10,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:45:10,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:45:11,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:13,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:15,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:15,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:45:17,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:19,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:20,469 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:45:21,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:45:21,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:23,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:25,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:45:25,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:45:25,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:45:25,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:26,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:45:27,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:29,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:31,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:31,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:45:33,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:35,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:37,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:45:37,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:39,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:40,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:45:40,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:45:40,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:45:41,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:42,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:45:42,586 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:45:42,587 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:45:42,587 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:45:42,589 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:45:42,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:45:43,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:45,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:47,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:47,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:45:49,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:50,459 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:45:51,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:53,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:45:53,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:55,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:45:55,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:45:55,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:45:55,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:57,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:45:58,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:45:59,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:01,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:03,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:03,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:46:05,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:07,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:09,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:46:09,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:10,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:46:10,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:46:10,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:46:11,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:13,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:14,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:46:15,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:17,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:19,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:19,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:46:20,461 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:46:21,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:23,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:25,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:46:25,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:46:25,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:46:25,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:46:25,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:27,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:46:27,135 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:46:27,136 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:46:27,137 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:46:27,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:27,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:46:29,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:30,642 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:46:31,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:33,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:35,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:35,973 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:46:37,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:39,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:40,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:46:40,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:46:40,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:46:41,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:46:41,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:43,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:45,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:46,631 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:46:47,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:49,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:50,463 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:46:51,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:51,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:46:53,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:55,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:46:55,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:46:55,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:46:55,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:57,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:46:57,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:46:59,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:01,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:02,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:47:03,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:05,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:07,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:08,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:47:09,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:10,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:47:10,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:47:10,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:47:11,682 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:47:11,683 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:47:11,684 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:47:11,685 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:47:11,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:11,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:47:13,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:47:13,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:15,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:17,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:18,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:47:19,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:20,465 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:47:21,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:23,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:24,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:47:25,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:47:25,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:47:25,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:47:25,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:27,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:29,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:47:29,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:31,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:33,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:34,767 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:47:35,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:37,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:39,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:40,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:47:40,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:47:40,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:47:40,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:47:41,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:43,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:45,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:47:45,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:47,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:49,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:50,468 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:47:50,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:47:51,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:53,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:55,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:47:55,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:47:55,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:47:55,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:56,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:47:56,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:47:56,255 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:47:56,256 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:47:56,257 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:47:56,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:47:57,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:47:59,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:01,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:48:02,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:04,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:06,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:06,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:48:08,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:10,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:10,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:48:10,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:48:10,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:48:12,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:12,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:48:14,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:16,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:17,553 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:48:18,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:20,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:20,470 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:48:22,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:22,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:48:24,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:25,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:48:25,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:48:25,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:48:26,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:28,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:28,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:48:30,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:32,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:33,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:48:34,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:36,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:38,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:38,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:48:40,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:40,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:48:40,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:48:40,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:48:40,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:48:40,766 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:48:40,766 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:48:40,767 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:48:41,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:41,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:48:44,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:44,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:48:46,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:48,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:49,620 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:48:50,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:50,472 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:48:52,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:54,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:54,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:48:55,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:48:55,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:48:55,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:48:56,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:48:58,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:00,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:49:00,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:02,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:04,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:05,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:49:06,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:08,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:10,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:10,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:49:10,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:49:10,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:49:10,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:49:12,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:14,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:15,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:16,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:49:17,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:19,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:20,474 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:49:21,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:21,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:49:23,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:25,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:49:25,258 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:49:25,259 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:49:25,261 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:49:25,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:25,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:49:25,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:49:25,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:49:25,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:49:26,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:49:27,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:29,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:31,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:32,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:49:33,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:35,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:37,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:37,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:49:39,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:40,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:49:40,647 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:49:40,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:49:41,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:43,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:49:43,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:45,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:47,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:48,476 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:49:49,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:50,476 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:49:51,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:53,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:53,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:49:55,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:55,664 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:49:55,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:49:55,666 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:49:57,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:49:59,141 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:49:59,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:01,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:03,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:04,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:50:05,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:07,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:09,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:09,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:50:09,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:50:09,869 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:50:09,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:50:09,871 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:50:10,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:50:10,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:50:10,651 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:50:10,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:50:11,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:13,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:15,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:50:15,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:17,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:19,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:20,478 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:50:20,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:50:21,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:23,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:25,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:25,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:50:25,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:50:25,651 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:50:25,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:50:27,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:29,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:31,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:50:31,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:33,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:35,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:36,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:50:37,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:39,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:40,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:50:40,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:50:40,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:50:41,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:41,762 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:50:43,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:45,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:47,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:50:47,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:49,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:50,480 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:50:51,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:52,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:50:53,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:54,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:50:54,512 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:50:54,513 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:50:54,515 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:50:55,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:55,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:50:55,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:50:55,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:50:55,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:50:57,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:50:58,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:50:59,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:01,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:03,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:51:03,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:05,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:07,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:08,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:51:09,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:10,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:51:10,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:51:10,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:51:11,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:13,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:14,143 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:51:15,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:17,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:19,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:19,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:51:20,483 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:51:21,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:23,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:24,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:51:25,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:25,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:51:25,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:51:25,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:51:27,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:29,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:30,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:51:31,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:33,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:35,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:35,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:51:37,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:39,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:51:39,219 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:51:39,219 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:51:39,221 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:51:39,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:39,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:51:40,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:51:40,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:51:40,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:51:40,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:51:41,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:43,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:45,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:46,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:51:47,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:49,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:50,485 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:51:51,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:51:51,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:53,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:55,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:55,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:51:55,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:51:55,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:51:56,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:51:57,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:51:59,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:01,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:02,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:52:03,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:05,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:07,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:07,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:52:09,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:10,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:52:10,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:52:10,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:52:11,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:13,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:52:13,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:15,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:17,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:18,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:52:19,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:20,487 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:52:21,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:23,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:23,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:52:23,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:52:23,718 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:52:23,718 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:52:23,721 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:52:24,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:52:25,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:25,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:52:25,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:52:25,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:52:27,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:29,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:52:29,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:31,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:33,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:34,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:52:35,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:37,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:39,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:39,674 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:52:40,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:52:40,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:52:40,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:52:41,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:43,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:44,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:52:45,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:47,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:49,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:50,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:52:50,489 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:52:51,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:53,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:55,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:55,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:52:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:52:55,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:52:55,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:52:57,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:52:59,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:00,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:53:01,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:03,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:05,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:06,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:53:07,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:08,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:53:08,160 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:53:08,161 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:53:08,163 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:53:08,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:53:09,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:10,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:53:10,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:53:10,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:53:11,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:11,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:53:13,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:15,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:17,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:53:17,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:19,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:20,492 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:53:21,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:22,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:53:23,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:25,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:53:25,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:53:25,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:53:27,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:27,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:53:29,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:31,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:33,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:53:33,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:35,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:37,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:38,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:53:39,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:53:40,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:53:40,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:53:41,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:43,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:43,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:53:45,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:47,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:49,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:53:49,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:50,493 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:53:51,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:52,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:53:52,659 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:53:52,659 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:53:52,661 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:53:53,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:53,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:53:54,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:53:55,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:53:55,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:53:55,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:53:57,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:59,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:53:59,738 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:54:01,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:03,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:05,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:54:05,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:07,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:09,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:10,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:54:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:54:10,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:54:10,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:54:11,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:13,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:15,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:15,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:54:17,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:19,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:20,496 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:54:21,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:54:21,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:23,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:54:25,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:54:25,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:25,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:54:26,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:54:27,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:29,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:31,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:31,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:54:33,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:35,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:37,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:54:37,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:54:37,247 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:54:37,248 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:54:37,249 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:54:37,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:37,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:54:39,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:54:40,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:54:40,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:54:41,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:42,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:54:43,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:45,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:47,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:47,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:54:49,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:50,498 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:54:51,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:53,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:54:53,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:54:55,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:54:55,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:54:55,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:57,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:54:58,522 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:54:59,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:01,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:03,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:03,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:55:05,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:07,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:09,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:55:09,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:55:10,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:55:10,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:55:11,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:13,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:14,567 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:55:15,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:17,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:19,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:19,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:55:20,500 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:55:21,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:21,738 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:55:21,740 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:55:21,740 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:55:21,742 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:55:22,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:55:23,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:25,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:55:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:55:25,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:55:25,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:55:25,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:27,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:29,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:30,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:55:31,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:33,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:35,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:35,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:55:37,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:39,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:40,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:55:40,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:55:40,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:55:41,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:55:41,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:43,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:45,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:46,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:55:47,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:49,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:50,502 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:55:51,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:51,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:55:53,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:55,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:55:55,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:55:55,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:55:55,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:57,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:55:57,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:55:59,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:01,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:02,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:56:03,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:05,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:06,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:56:06,193 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:56:06,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:56:06,196 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:56:06,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:56:07,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:07,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:56:09,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:10,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:56:10,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:56:10,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:56:11,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:13,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:56:13,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:15,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:17,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:18,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:56:19,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:20,505 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:56:21,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:23,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:24,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:56:25,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:56:25,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:56:25,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:56:25,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:27,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:29,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:56:29,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:31,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:33,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:34,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:56:35,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:37,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:39,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:40,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:56:40,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:56:40,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:56:40,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:56:41,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:43,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:45,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:56:45,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:47,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:49,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:50,507 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:56:50,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:56:50,818 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:56:50,820 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:56:50,821 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:56:50,822 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:56:50,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:56:51,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:53,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:55,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:56:55,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:56:55,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:56:55,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:56:55,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:57,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:56:59,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:00,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:57:01,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:03,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:05,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:06,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:57:07,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:09,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:10,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:57:10,655 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:57:10,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:57:11,732 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:57:11,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:13,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:15,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:16,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:57:17,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:19,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:20,509 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:57:21,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:22,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:57:23,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:25,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:57:25,655 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:57:25,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:57:25,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:27,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:57:27,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:29,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:31,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:32,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:57:33,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:35,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:57:35,535 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:57:35,536 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:57:35,537 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:57:35,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:35,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:57:37,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:38,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:57:39,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:40,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:57:40,655 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:57:40,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:57:41,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:43,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:57:43,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:45,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:47,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:48,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:57:49,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:50,511 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:57:51,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:53,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:54,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:57:55,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:57:55,655 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:57:55,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:57:55,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:57,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:57:59,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:57:59,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:01,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:03,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:05,030 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:58:05,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:07,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:09,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:10,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:58:10,655 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:58:10,655 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:58:10,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:58:11,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:13,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:15,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:58:15,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:17,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:19,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:20,228 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:58:20,230 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:58:20,231 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:58:20,233 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:58:20,512 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:58:20,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:58:21,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:58:21,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:23,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:25,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:58:25,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:58:25,667 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:58:25,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:26,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:58:27,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:29,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:31,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:58:32,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:34,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:36,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:37,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:58:38,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:40,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:40,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:58:40,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:58:40,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:58:42,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:42,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:58:44,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:46,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:48,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:48,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:58:50,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:50,515 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:58:52,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:53,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:58:54,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:55,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:58:55,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:58:55,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:58:56,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:58,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:58:59,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:59:00,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:02,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:04,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:04,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:59:05,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:59:05,360 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:59:05,361 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:59:05,362 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:59:06,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:06,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:59:08,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:09,949 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:59:10,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:10,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:59:10,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:59:10,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:59:12,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:14,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:15,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:59:16,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:18,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:20,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:20,517 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:59:20,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:59:22,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:24,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:25,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:59:25,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:59:25,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:59:25,815 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:59:26,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:28,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:30,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:30,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:59:32,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:34,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:36,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:36,537 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:59:38,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:40,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:40,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:59:40,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:59:40,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:59:41,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:59:42,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:44,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:46,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:46,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:59:48,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:50,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:50,531 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 19:59:51,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 19:59:51,381 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 19:59:51,381 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 19:59:51,383 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 19:59:52,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:52,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 19:59:52,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:59:54,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:55,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 19:59:55,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 19:59:55,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 19:59:56,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 19:59:57,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 19:59:58,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:00,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:02,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:03,603 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:00:04,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:06,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:08,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:09,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:00:10,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:10,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:00:10,661 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:00:10,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:00:12,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:14,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:14,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:00:16,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:18,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:19,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:00:20,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:20,521 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:00:22,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:24,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:25,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:00:25,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:00:25,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:00:25,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:00:28,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:30,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:30,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:00:32,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:34,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:36,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:00:36,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:38,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:40,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:40,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:00:40,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:00:40,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:00:41,109 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:00:41,111 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:00:41,111 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:00:41,113 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:00:41,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:41,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:00:42,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:00:44,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:46,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:47,865 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:00:48,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:50,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:50,523 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:00:52,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:53,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:00:54,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:55,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:00:55,661 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:00:55,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:00:56,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:58,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:00:59,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:01:00,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:02,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:04,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:05,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:01:06,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:08,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:10,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:10,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:01:10,661 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:01:10,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:01:10,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:01:12,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:14,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:15,767 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:01:16,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:18,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:20,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:20,526 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:01:21,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:01:22,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:24,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:25,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:01:25,661 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:01:25,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:01:26,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:26,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:01:28,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:29,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:01:29,223 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:01:29,223 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:01:29,225 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:01:29,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:29,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:01:32,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:01:32,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:34,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:36,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:37,805 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:01:38,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:40,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:40,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:01:40,805 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:01:40,805 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:01:42,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:43,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:01:44,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:46,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:48,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:49,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:01:50,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:50,539 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:01:52,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:54,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:54,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:01:55,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:01:55,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:01:55,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:01:56,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:01:58,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:00,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:00,748 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:02:02,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:04,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:06,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:06,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:02:08,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:10,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:10,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:02:10,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:02:10,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:02:12,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:02:12,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:14,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:16,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:16,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:02:16,847 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:02:16,847 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:02:16,849 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:02:17,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:17,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:02:17,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:02:20,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:20,530 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:02:22,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:23,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:02:24,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:25,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:02:25,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:02:25,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:02:26,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:28,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:29,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:02:30,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:32,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:34,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:34,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:02:36,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:38,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:40,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:40,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:02:40,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:02:40,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:02:40,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:02:42,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:44,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:46,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:02:46,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:48,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:50,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:50,532 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:02:51,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:02:52,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:54,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:55,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:02:55,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:02:55,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:02:56,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:02:57,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:02:58,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:00,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:02,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:03,111 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:03:04,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:03:04,019 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:03:04,020 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:03:04,023 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:03:04,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:04,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:03:05,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:08,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:08,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:03:10,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:10,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:03:10,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:03:10,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:03:12,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:14,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:03:14,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:16,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:18,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:20,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:03:20,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:20,535 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:03:22,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:24,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:25,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:03:25,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:03:25,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:03:25,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:03:26,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:28,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:30,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:31,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:03:32,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:34,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:36,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:36,948 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:03:38,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:40,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:40,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:03:40,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:03:40,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:03:42,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:42,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:03:44,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:46,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:48,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:03:48,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:50,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:50,549 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:03:50,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:03:50,953 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:03:50,953 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:03:50,955 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:03:51,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:51,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:03:53,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:03:54,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:55,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:03:55,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:03:55,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:03:56,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:58,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:03:59,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:04:00,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:02,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:04,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:05,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:04:06,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:08,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:10,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:10,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:04:10,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:04:10,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:04:10,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:04:12,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:14,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:16,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:04:16,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:18,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:20,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:20,540 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:04:21,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:04:22,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:24,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:25,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:04:25,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:04:25,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:04:26,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:27,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:04:28,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:30,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:32,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:33,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:04:34,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:36,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:37,620 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:04:37,622 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:04:37,622 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:04:37,624 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:04:38,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:38,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:04:38,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:04:39,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:40,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:04:40,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:04:40,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:04:42,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:44,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:04:44,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:46,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:48,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:49,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:04:50,542 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:04:50,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:52,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:54,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:55,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:04:55,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:04:55,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:04:55,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:04:56,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:04:58,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:00,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:00,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:05:02,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:04,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:06,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:06,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:05:08,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:10,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:10,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:05:10,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:05:10,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:05:12,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:05:12,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:14,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:16,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:17,748 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:05:18,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:20,544 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:05:20,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:22,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:23,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:05:24,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:05:24,173 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:05:24,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:05:24,175 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:05:24,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:24,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:05:25,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:25,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:05:25,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:05:25,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:05:28,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:28,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:05:30,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:32,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:34,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:05:34,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:36,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:38,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:39,938 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:05:40,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:40,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:05:40,787 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:05:40,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:05:42,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:44,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:45,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:05:46,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:48,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:50,556 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:05:50,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:05:50,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:52,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:54,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:55,692 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:05:55,818 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:05:55,818 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:05:55,819 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:05:56,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:05:58,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:00,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:01,275 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:06:02,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:04,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:06,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:06,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:06:08,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:10,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:06:10,548 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:06:10,549 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:06:10,550 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:06:10,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:10,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:06:10,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:06:10,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:06:10,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:06:11,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:11,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:06:14,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:16,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:17,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:06:18,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:19,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:20,548 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:06:21,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:22,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:06:23,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:25,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:25,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:06:25,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:06:25,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:06:27,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:28,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:06:29,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:31,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:33,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:06:33,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:35,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:37,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:39,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:06:39,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:40,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:06:40,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:06:40,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:06:41,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:43,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:44,686 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:06:45,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:47,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:49,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:50,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:06:50,551 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:06:51,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:53,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:55,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:55,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:06:55,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:06:55,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:06:55,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:06:56,706 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:06:56,708 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:06:56,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:06:56,710 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:06:57,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:06:57,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:06:59,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:01,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:07:01,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:03,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:05,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:06,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:07:07,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:09,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:10,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:07:10,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:07:10,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:07:11,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:12,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:07:13,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:15,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:17,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:17,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:07:19,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:20,553 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:07:21,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:23,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:07:23,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:25,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:25,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:07:25,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:07:25,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:07:27,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:28,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:07:29,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:31,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:33,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:34,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:07:35,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:37,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:39,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:39,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:07:40,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:07:40,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:07:40,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:07:41,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:42,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:07:42,667 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:07:42,667 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:07:42,669 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:07:42,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:07:43,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:45,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:07:45,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:47,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:49,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:50,567 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:07:50,569 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:07:51,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:53,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:55,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:55,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:07:55,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:07:55,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:07:55,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:07:57,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:07:59,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:00,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:08:01,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:03,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:05,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:06,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:08:07,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:09,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:10,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:08:10,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:08:10,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:08:11,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:11,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:08:13,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:15,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:17,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:08:17,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:19,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:20,558 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:08:21,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:22,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:08:23,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:25,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:25,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:08:25,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:08:25,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:08:27,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:28,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:08:28,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:08:28,454 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:08:28,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:08:28,455 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:08:28,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:08:29,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:31,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:33,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:33,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:08:35,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:37,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:39,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:08:39,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:40,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:08:40,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:08:40,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:08:41,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:43,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:44,921 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:08:45,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:47,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:49,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:50,427 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:08:50,560 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:08:51,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:53,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:55,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:08:55,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:08:55,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:08:55,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:08:55,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:57,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:08:59,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:01,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:09:01,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:03,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:05,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:06,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:09:07,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:09,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:10,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:09:10,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:09:10,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:09:11,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:09:11,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:13,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:14,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:09:14,229 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:09:14,229 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:09:14,231 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:09:14,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:09:16,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:17,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:09:18,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:20,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:20,562 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:09:22,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:22,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:09:24,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:25,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:09:25,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:09:25,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:09:26,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:27,948 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:09:28,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:30,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:32,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:33,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:09:34,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:36,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:38,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:38,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:09:40,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:40,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:09:40,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:09:40,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:09:42,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:44,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:44,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:09:46,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:48,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:50,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:09:50,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:50,575 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:09:52,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:54,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:55,569 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:09:55,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:09:55,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:09:55,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:09:56,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:09:58,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:00,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:00,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:10:00,092 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:10:00,092 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:10:00,094 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:10:01,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:10:01,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:10:02,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:04,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:06,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:06,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:10:08,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:10,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:10,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:10:10,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:10:10,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:10:12,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:10:12,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:14,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:16,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:17,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:10:18,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:20,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:20,567 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:10:22,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:22,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:10:24,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:25,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:10:25,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:10:25,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:10:26,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:28,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:28,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:10:30,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:32,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:33,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:10:34,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:36,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:38,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:39,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:10:40,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:40,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:10:40,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:10:40,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:10:42,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:44,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:44,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:10:45,635 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:10:45,637 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:10:45,638 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:10:45,639 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:10:46,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:46,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:10:48,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:50,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:50,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:10:50,568 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:10:52,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:54,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:55,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:10:55,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:10:55,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:10:55,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:10:56,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:10:58,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:00,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:01,185 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:11:02,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:04,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:06,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:06,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:11:08,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:10,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:10,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:11:10,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:11:10,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:11:12,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:11:12,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:14,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:16,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:17,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:11:18,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:20,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:20,571 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:11:22,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:23,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:11:24,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:25,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:11:25,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:11:25,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:11:26,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:28,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:28,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:11:30,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:31,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:11:31,323 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:11:31,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:11:31,325 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:11:32,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:32,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:11:34,117 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:11:34,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:36,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:38,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:39,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:11:40,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:40,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:11:40,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:11:40,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:11:42,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:44,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:45,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:11:46,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:48,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:50,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:50,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:11:50,585 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:11:52,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:54,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:55,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:11:55,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:11:55,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:11:55,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:11:56,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:11:58,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:00,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:01,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:12:02,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:04,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:06,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:06,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:12:08,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:10,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:10,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:12:10,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:12:10,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:12:12,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:12,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:12:14,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:16,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:16,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:12:16,875 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:12:16,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:12:16,877 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:12:17,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:12:17,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:12:18,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:20,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:20,620 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:12:22,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:23,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:12:24,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:25,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:12:25,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:12:25,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:12:26,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:28,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:28,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:12:30,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:32,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:34,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:12:34,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:36,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:38,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:39,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:12:40,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:40,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:12:40,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:12:40,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:12:42,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:44,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:45,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:12:46,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:48,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:50,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:50,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:12:50,622 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:12:52,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:54,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:55,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:12:55,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:12:55,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:12:55,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:12:56,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:12:58,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:00,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:01,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:13:02,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:02,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:13:02,319 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:13:02,319 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:13:02,321 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:13:03,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:13:04,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:06,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:06,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:13:08,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:10,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:10,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:13:10,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:13:10,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:13:12,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:12,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:13:14,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:16,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:17,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:13:18,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:20,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:20,624 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:13:22,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:23,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:13:24,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:25,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:13:25,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:13:25,802 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:13:26,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:28,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:28,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:13:30,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:32,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:34,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:13:34,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:36,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:38,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:39,635 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:13:40,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:40,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:13:40,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:13:40,802 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:13:42,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:44,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:45,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:13:46,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:47,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:13:47,738 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:13:47,739 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:13:47,741 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:13:48,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:48,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:13:50,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:50,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:13:50,638 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:13:52,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:54,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:55,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:13:55,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:13:55,802 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:13:55,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:13:56,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:13:58,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:00,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:01,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:14:02,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:04,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:06,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:06,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:14:08,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:10,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:10,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:14:10,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:14:10,802 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:14:12,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:14:12,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:14,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:16,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:17,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:14:18,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:20,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:20,631 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:14:22,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:23,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:14:24,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:25,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:14:25,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:14:25,802 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:14:26,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:28,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:28,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:14:30,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:32,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:33,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:14:33,110 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:14:33,110 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:14:33,112 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:14:33,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:14:34,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:14:34,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:36,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:38,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:39,538 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:14:40,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:40,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:14:40,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:14:40,802 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:14:42,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:44,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:45,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:14:46,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:48,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:50,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:50,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:14:50,632 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:14:52,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:54,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:55,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:14:55,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:14:55,802 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:14:55,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:14:56,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:14:58,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:00,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:01,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:15:02,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:04,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:06,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:06,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:15:08,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:10,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:10,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:15:10,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:15:10,803 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:15:12,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:15:12,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:14,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:16,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:17,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:15:18,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:18,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:15:18,572 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:15:18,573 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:15:18,574 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:15:19,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:19,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:15:20,634 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:15:22,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:23,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:15:24,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:25,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:15:25,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:15:25,803 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:15:26,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:28,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:28,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:15:30,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:32,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:34,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:15:34,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:36,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:38,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:39,595 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:15:40,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:40,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:15:40,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:15:40,803 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:15:42,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:44,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:45,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:15:46,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:48,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:50,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:50,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:15:50,649 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:15:52,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:54,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:55,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:15:55,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:15:55,803 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:15:55,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:15:56,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:15:58,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:00,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:01,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:16:02,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:04,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:16:04,011 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:16:04,012 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:16:04,014 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:16:04,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:04,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:16:05,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:06,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:16:08,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:10,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:10,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:16:10,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:16:10,803 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:16:12,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:16:12,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:14,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:16,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:17,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:16:18,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:20,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:20,638 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:16:22,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:23,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:16:24,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:25,853 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:16:25,854 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:16:25,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:16:26,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:28,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:28,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:16:30,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:32,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:34,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:16:34,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:36,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:38,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:39,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:16:40,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:40,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:16:40,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:16:40,848 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:16:42,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:44,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:44,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:16:46,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:48,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:49,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:16:49,368 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:16:49,368 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:16:49,370 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:16:49,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:49,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:16:50,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:16:50,641 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:16:52,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:54,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:55,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:16:55,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:16:55,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:16:55,848 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:16:56,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:16:58,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:00,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:01,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:17:02,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:04,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:06,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:06,674 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:17:08,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:10,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:10,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:17:10,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:17:10,848 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:17:12,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:17:12,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:14,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:16,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:17,592 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:17:18,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:20,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:20,643 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:17:22,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:23,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:17:24,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:25,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:17:25,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:17:25,848 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:17:26,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:28,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:17:28,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:30,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:32,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:33,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:17:34,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:34,732 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:17:34,734 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:17:34,734 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:17:34,736 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:17:35,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:35,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:17:38,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:39,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:17:40,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:40,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:17:40,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:17:40,848 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:17:42,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:44,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:44,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:17:46,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:48,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:50,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:17:50,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:50,657 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:17:52,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:54,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:55,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:17:55,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:17:55,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:17:55,848 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:17:56,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:17:58,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:00,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:01,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:18:02,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:04,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:06,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:18:06,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:08,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:10,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:10,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:18:10,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:18:10,849 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:18:11,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:18:12,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:14,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:16,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:17,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:18:18,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:20,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:18:20,085 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:18:20,085 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:18:20,087 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:18:20,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:20,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:18:20,647 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:18:21,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:22,865 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:18:24,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:25,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:18:25,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:18:25,849 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:18:26,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:28,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:18:28,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:30,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:32,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:33,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:18:34,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:36,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:38,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:39,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:18:40,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:40,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:18:40,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:18:40,849 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:18:42,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:44,551 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:18:44,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:46,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:48,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:49,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:18:50,649 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:18:50,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:52,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:54,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:55,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:18:55,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:18:55,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:18:55,849 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:18:56,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:18:58,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:00,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:00,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:19:02,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:04,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:05,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:19:05,373 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:19:05,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:19:05,375 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:19:05,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:05,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:19:06,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:19:08,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:10,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:10,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:19:10,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:19:10,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:19:11,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:19:12,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:14,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:16,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:17,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:19:18,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:20,651 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:19:20,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:22,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:19:22,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:24,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:25,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:19:25,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:19:25,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:19:26,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:28,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:19:28,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:30,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:32,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:33,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:19:34,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:36,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:38,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:38,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:19:40,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:19:40,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:19:40,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:19:40,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:41,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:43,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:44,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:19:45,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:47,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:49,836 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:19:49,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:50,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:19:50,728 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:19:50,728 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:19:50,728 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:19:50,730 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:19:50,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:19:51,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:53,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:55,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:19:55,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:19:55,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:19:55,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:19:55,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:57,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:19:59,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:00,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:20:01,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:03,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:05,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:06,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:20:07,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:09,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:10,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:20:10,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:20:10,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:20:11,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:11,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:20:13,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:15,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:17,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:20:17,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:19,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:20,656 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:20:21,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:22,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:20:23,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:25,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:20:25,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:20:25,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:20:25,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:27,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:28,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:20:29,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:31,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:33,499 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:20:33,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:35,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:36,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:20:36,177 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:20:36,177 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:20:36,179 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:20:36,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:20:37,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:38,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:20:39,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:40,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:20:40,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:20:40,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:20:41,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:43,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:44,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:20:45,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:47,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:49,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:20:49,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:50,658 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:20:51,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:53,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:55,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:20:55,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:20:55,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:20:55,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:20:55,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:57,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:20:59,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:00,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:21:01,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:03,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:05,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:06,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:21:07,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:09,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:10,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:21:10,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:21:10,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:21:11,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:21:11,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:13,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:15,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:17,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:21:18,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:20,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:20,660 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:21:21,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:21:21,657 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:21:21,658 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:21:21,659 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:21:22,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:22,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:21:22,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:21:24,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:25,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:21:25,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:21:25,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:21:26,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:28,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:28,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:21:30,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:32,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:33,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:21:34,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:36,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:38,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:39,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:21:40,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:40,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:21:40,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:21:40,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:21:42,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:44,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:44,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:21:46,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:48,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:50,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:21:50,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:50,711 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:21:52,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:54,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:55,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:21:55,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:21:55,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:21:55,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:21:56,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:21:58,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:00,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:00,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:22:02,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:04,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:06,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:06,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:22:07,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:22:07,273 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:22:07,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:22:07,275 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:22:08,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:08,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:22:10,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:10,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:22:10,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:22:10,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:22:12,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:22:12,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:14,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:16,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:17,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:22:18,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:20,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:20,665 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:22:22,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:22,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:22:24,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:25,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:22:25,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:22:25,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:22:26,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:28,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:28,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:22:30,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:32,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:33,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:22:34,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:36,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:38,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:39,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:22:40,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:40,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:22:40,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:22:40,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:22:42,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:44,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:44,663 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:22:46,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:48,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:50,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:50,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:22:50,666 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:22:52,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:52,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:22:52,793 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:22:52,793 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:22:52,795 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:22:53,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:22:54,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:55,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:22:55,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:22:55,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:22:55,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:22:56,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:22:58,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:00,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:01,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:23:02,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:04,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:06,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:06,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:23:08,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:10,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:10,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:23:10,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:23:10,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:23:12,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:23:12,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:14,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:16,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:17,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:23:18,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:20,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:20,669 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:23:22,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:22,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:23:24,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:25,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:23:25,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:23:25,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:23:26,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:28,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:28,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:23:30,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:32,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:33,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:23:34,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:36,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:38,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:38,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:23:38,381 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:23:38,381 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:23:38,383 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:23:39,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:23:39,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:23:40,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:40,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:23:40,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:23:40,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:23:42,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:44,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:44,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:23:46,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:48,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:50,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:50,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:23:50,670 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:23:52,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:54,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:55,674 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:23:55,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:23:55,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:23:55,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:23:56,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:23:58,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:00,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:01,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:24:02,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:04,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:06,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:06,551 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:24:08,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:10,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:10,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:24:10,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:24:10,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:24:12,015 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:24:12,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:14,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:16,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:17,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:24:18,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:20,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:20,673 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:24:22,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:22,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:24:23,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:24:23,814 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:24:23,815 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:24:23,818 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:24:24,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:24,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:24:25,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:24:25,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:24:25,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:24:26,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:28,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:28,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:24:30,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:32,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:33,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:24:34,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:36,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:38,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:39,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:24:40,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:40,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:24:40,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:24:40,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:24:42,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:44,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:44,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:24:46,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:48,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:50,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:50,238 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:24:50,675 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:24:52,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:54,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:55,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:24:55,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:24:55,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:24:55,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:24:56,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:24:58,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:00,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:01,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:25:02,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:04,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:06,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:06,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:25:08,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:09,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:25:09,275 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:25:09,275 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:25:09,277 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:25:10,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:10,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:25:10,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:25:10,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:25:10,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:25:12,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:25:12,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:14,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:16,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:17,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:25:18,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:20,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:20,677 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:25:22,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:22,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:25:24,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:25,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:25:25,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:25:25,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:25:26,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:28,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:28,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:25:30,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:32,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:33,836 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:25:34,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:36,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:38,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:39,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:25:40,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:40,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:25:40,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:25:40,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:25:42,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:44,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:44,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:25:46,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:48,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:50,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:25:50,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:50,680 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:25:52,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:54,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:54,686 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:25:54,688 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:25:54,688 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:25:54,690 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:25:55,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:25:55,727 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:25:55,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:25:55,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:25:55,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:25:56,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:25:58,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:00,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:01,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:26:02,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:04,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:06,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:06,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:26:08,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:10,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:10,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:26:10,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:26:10,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:26:12,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:26:12,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:14,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:16,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:17,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:26:18,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:20,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:20,682 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:26:22,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:23,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:26:24,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:25,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:26:25,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:26:25,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:26:26,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:28,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:28,530 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:26:30,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:32,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:34,045 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:26:34,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:36,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:38,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:39,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:26:40,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:40,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:26:40,404 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:26:40,405 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:26:40,406 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:26:40,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:26:40,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:26:40,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:26:41,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:26:42,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:44,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:45,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:26:46,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:48,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:50,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:50,511 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:26:50,684 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:26:52,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:54,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:55,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:26:55,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:26:55,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:26:55,949 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:26:56,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:26:58,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:00,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:01,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:27:02,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:04,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:06,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:06,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:27:08,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:10,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:10,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:27:10,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:27:10,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:27:12,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:12,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:27:14,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:16,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:17,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:27:18,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:20,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:20,686 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:27:22,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:23,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:27:24,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:25,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:27:25,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:27:25,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:27:26,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:27:26,137 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:27:26,137 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:27:26,139 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:27:26,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:26,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:27:28,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:28,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:27:30,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:32,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:34,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:34,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:27:36,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:38,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:39,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:27:40,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:40,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:27:40,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:27:40,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:27:42,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:44,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:45,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:27:46,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:48,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:50,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:50,690 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:27:50,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:27:52,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:54,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:55,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:27:55,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:27:55,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:27:56,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:27:56,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:27:58,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:00,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:01,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:28:02,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:04,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:06,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:07,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:28:08,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:10,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:10,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:28:10,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:28:10,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:28:11,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:28:11,779 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:28:11,779 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:28:11,781 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:28:12,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:12,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:28:12,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:28:14,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:16,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:18,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:28:18,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:20,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:20,692 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:28:22,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:23,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:28:24,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:25,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:28:25,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:28:25,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:28:26,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:28,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:29,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:28:30,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:32,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:34,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:34,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:28:36,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:38,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:40,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:28:40,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:40,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:28:40,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:28:40,901 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:28:42,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:44,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:45,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:28:46,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:48,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:50,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:50,694 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:28:50,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:28:52,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:54,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:55,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:28:55,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:28:55,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:28:55,973 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:28:56,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:57,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:28:57,315 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:28:57,315 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:28:57,317 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:28:57,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:28:58,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:28:59,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:00,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:01,921 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:29:02,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:04,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:06,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:07,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:29:08,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:10,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:10,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:29:10,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:29:10,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:29:12,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:12,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:29:14,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:16,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:18,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:29:18,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:20,697 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:29:20,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:22,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:23,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:29:24,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:25,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:29:25,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:29:25,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:29:26,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:28,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:29,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:29:30,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:32,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:34,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:29:34,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:36,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:38,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:40,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:29:40,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:40,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:29:40,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:29:40,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:29:42,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:42,782 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:29:42,783 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:29:42,783 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:29:42,785 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:29:43,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:43,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:29:44,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:45,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:29:46,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:48,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:50,699 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:29:50,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:29:50,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:52,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:54,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:55,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:29:55,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:29:55,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:29:56,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:29:56,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:29:58,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:00,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:01,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:30:02,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:04,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:06,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:07,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:30:08,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:10,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:10,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:30:10,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:30:10,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:30:12,789 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:30:12,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:14,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:16,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:18,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:30:18,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:20,701 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:30:20,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:22,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:23,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:30:24,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:25,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:30:25,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:30:25,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:30:26,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:28,109 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:30:28,111 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:30:28,111 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:30:28,113 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:30:28,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:28,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:30:29,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:30:29,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:30,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:32,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:34,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:30:34,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:36,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:38,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:39,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:30:40,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:40,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:30:40,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:30:40,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:30:42,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:44,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:45,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:30:46,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:48,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:50,703 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:30:50,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:30:50,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:52,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:54,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:55,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:30:55,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:30:55,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:30:55,950 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:30:56,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:30:58,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:00,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:01,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:31:02,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:04,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:06,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:07,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:31:08,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:10,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:10,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:31:10,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:31:10,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:31:12,627 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:31:12,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:13,470 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:31:13,471 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:31:13,472 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:31:13,473 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:31:13,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:31:14,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:15,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:16,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:18,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:31:18,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:20,705 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:31:20,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:22,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:23,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:31:24,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:25,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:31:25,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:31:25,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:31:26,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:28,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:28,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:31:30,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:32,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:34,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:31:34,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:36,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:38,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:39,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:31:40,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:31:40,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:31:40,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:31:40,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:42,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:44,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:45,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:31:46,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:48,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:50,699 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:31:50,707 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:31:50,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:52,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:54,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:55,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:31:55,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:31:55,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:31:55,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:31:56,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:58,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:31:58,850 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:31:58,850 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:31:58,852 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:31:58,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:31:58,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:31:59,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:00,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:01,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:32:02,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:04,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:06,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:07,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:32:08,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:10,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:32:10,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:32:10,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:32:10,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:12,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:32:12,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:14,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:16,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:17,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:32:18,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:20,709 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:32:20,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:22,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:23,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:32:24,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:25,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:32:25,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:32:25,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:32:26,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:28,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:32:28,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:30,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:32,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:34,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:32:34,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:36,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:38,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:39,655 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:32:40,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:32:40,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:32:40,873 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:32:40,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:42,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:44,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:32:44,129 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:32:44,130 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:32:44,132 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:32:44,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:44,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:32:45,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:32:45,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:46,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:48,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:50,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:32:50,712 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:32:50,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:52,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:54,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:55,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:32:55,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:32:55,873 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:32:56,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:32:56,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:32:58,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:00,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:01,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:33:02,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:04,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:06,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:33:06,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:08,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:10,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:33:10,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:33:10,873 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:33:10,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:12,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:33:12,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:14,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:16,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:17,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:33:18,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:20,714 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:33:20,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:22,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:23,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:33:24,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:25,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:33:25,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:33:25,894 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:33:26,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:28,595 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:33:28,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:29,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:33:29,439 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:33:29,440 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:33:29,441 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:33:29,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:33:30,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:32,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:33,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:34,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:33:35,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:37,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:39,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:39,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:33:40,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:33:40,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:33:40,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:33:41,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:43,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:44,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:33:45,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:47,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:49,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:50,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:33:50,716 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:33:51,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:53,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:55,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:55,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:33:55,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:33:55,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:33:55,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:33:57,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:33:59,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:01,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:01,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:34:03,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:05,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:06,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:34:07,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:09,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:10,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:34:10,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:34:10,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:34:11,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:12,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:34:13,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:14,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:34:14,714 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:34:14,714 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:34:14,716 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:34:15,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:15,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:34:16,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:17,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:17,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:34:19,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:20,718 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:34:21,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:22,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:34:23,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:25,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:25,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:34:25,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:34:25,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:34:27,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:28,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:34:29,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:31,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:33,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:33,758 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:34:35,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:37,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:39,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:39,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:34:40,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:34:40,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:34:40,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:34:41,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:43,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:44,618 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:34:45,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:47,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:49,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:50,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:34:50,721 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:34:51,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:53,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:55,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:55,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:34:55,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:34:55,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:34:55,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:34:57,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:59,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:34:59,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:34:59,919 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:34:59,919 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:34:59,922 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:35:00,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:35:00,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:35:01,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:02,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:03,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:05,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:06,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:35:07,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:09,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:10,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:35:10,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:35:10,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:35:11,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:12,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:35:13,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:15,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:17,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:17,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:35:19,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:20,722 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:35:21,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:22,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:35:23,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:25,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:25,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:35:25,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:35:25,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:35:27,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:28,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:35:29,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:31,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:33,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:33,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:35:35,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:37,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:38,896 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:35:39,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:40,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:35:40,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:35:40,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:35:41,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:43,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:44,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:35:45,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:45,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:35:45,159 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:35:45,159 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:35:45,161 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:35:46,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:35:47,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:48,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:49,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:49,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:35:50,725 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:35:51,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:53,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:55,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:35:55,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:55,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:35:55,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:35:55,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:35:57,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:35:59,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:00,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:36:01,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:03,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:05,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:05,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:36:07,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:09,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:10,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:36:10,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:36:10,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:36:11,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:36:11,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:13,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:15,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:16,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:36:17,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:19,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:20,742 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:36:21,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:22,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:36:23,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:25,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:25,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:36:25,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:36:25,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:36:27,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:27,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:36:29,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:30,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:36:30,299 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:36:30,299 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:36:30,302 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:36:31,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:31,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:36:32,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:33,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:36:34,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:36,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:38,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:38,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:36:40,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:40,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:36:40,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:36:40,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:36:42,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:43,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:36:44,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:46,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:48,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:49,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:36:50,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:50,730 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:36:52,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:54,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:54,739 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:36:55,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:36:55,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:36:55,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:36:56,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:36:58,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:00,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:37:00,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:02,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:04,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:05,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:37:06,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:08,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:10,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:10,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:37:10,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:37:10,877 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:37:10,991 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:37:12,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:14,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:15,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:37:15,500 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:37:15,501 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:37:15,502 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:37:16,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:16,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:37:16,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:37:18,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:20,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:20,732 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:37:21,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:37:22,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:24,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:25,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:37:25,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:37:25,877 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:37:26,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:27,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:37:28,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:30,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:32,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:32,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:37:34,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:36,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:38,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:37:38,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:40,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:40,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:37:40,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:37:40,880 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:37:42,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:43,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:37:44,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:46,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:48,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:48,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:37:50,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:50,734 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:37:52,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:54,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:54,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:37:55,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:37:55,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:37:55,880 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:37:56,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:58,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:37:59,795 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:38:00,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:00,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:38:00,647 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:38:00,647 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:38:00,649 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:38:01,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:38:02,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:04,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:05,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:38:06,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:08,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:10,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:10,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:38:10,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:38:10,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:38:10,880 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:38:12,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:14,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:16,156 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:38:16,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:18,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:20,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:20,737 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:38:21,738 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:38:22,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:24,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:25,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:38:25,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:38:25,880 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:38:26,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:27,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:38:28,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:30,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:32,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:32,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:38:34,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:36,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:37,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:38:38,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:40,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:40,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:38:40,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:38:40,880 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:38:42,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:43,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:38:44,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:45,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:38:45,908 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:38:45,909 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:38:45,911 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:38:46,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:46,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:38:48,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:48,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:38:50,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:50,739 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:38:52,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:54,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:38:54,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:55,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:38:55,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:38:55,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:38:56,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:58,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:38:59,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:39:00,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:02,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:04,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:04,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:39:06,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:08,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:10,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:39:10,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:10,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:39:10,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:39:10,880 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:39:12,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:14,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:15,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:39:16,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:18,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:20,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:20,741 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:39:21,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:39:22,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:24,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:25,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:39:25,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:39:25,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:39:26,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:27,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:39:28,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:30,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:31,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:39:31,117 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:39:31,117 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:39:31,119 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:39:31,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:39:32,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:39:32,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:34,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:36,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:37,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:39:38,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:40,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:40,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:39:40,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:39:40,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:39:42,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:42,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:39:44,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:46,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:48,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:39:48,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:50,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:50,743 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:39:52,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:53,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:39:54,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:55,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:39:55,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:39:55,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:39:56,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:58,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:39:59,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:40:00,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:02,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:04,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:40:04,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:06,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:08,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:09,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:40:10,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:10,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:40:10,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:40:10,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:40:12,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:14,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:15,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:40:16,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:40:16,229 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:40:16,230 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:40:16,231 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:40:16,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:16,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:40:17,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:18,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:20,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:20,745 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:40:20,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:40:22,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:24,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:25,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:40:25,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:40:25,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:40:25,950 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:40:26,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:28,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:30,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:31,639 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:40:32,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:34,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:36,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:37,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:40:38,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:40,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:40,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:40:40,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:40:40,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:40:42,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:40:42,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:44,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:46,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:47,852 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:40:48,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:50,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:50,748 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:40:52,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:53,263 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:40:54,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:55,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:40:55,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:40:55,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:40:56,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:40:58,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:40:58,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:00,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:01,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:41:01,337 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:41:01,338 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:41:01,339 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:41:01,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:41:02,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:03,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:04,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:41:04,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:06,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:08,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:09,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:41:10,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:10,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:41:10,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:41:10,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:41:12,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:14,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:14,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:41:16,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:18,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:20,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:41:20,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:20,750 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:41:22,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:24,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:25,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:41:25,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:41:25,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:41:25,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:41:26,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:28,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:30,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:31,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:41:32,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:34,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:36,569 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:41:36,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:38,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:40,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:40,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:41:40,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:41:40,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:41:42,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:41:42,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:44,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:46,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:41:46,419 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:41:46,419 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:41:46,420 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:41:46,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:46,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:41:47,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:41:47,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:48,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:50,752 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:41:50,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:52,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:52,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:41:54,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:55,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:41:55,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:41:55,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:41:56,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:41:58,260 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:41:58,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:00,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:02,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:03,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:42:04,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:06,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:08,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:09,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:42:10,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:10,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:42:10,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:42:10,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:42:12,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:14,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:42:14,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:16,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:18,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:20,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:42:20,767 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:42:20,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:22,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:24,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:25,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:42:25,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:42:25,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:42:25,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:42:26,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:28,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:30,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:30,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:42:31,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:42:31,668 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:42:31,668 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:42:31,670 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:42:31,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:42:32,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:33,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:34,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:36,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:42:36,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:38,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:40,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:40,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:42:40,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:42:40,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:42:42,065 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:42:42,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:44,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:46,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:47,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:42:48,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:50,757 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:42:50,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:52,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:53,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:42:54,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:55,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:42:55,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:42:55,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:42:56,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:42:58,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:42:58,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:00,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:02,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:04,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:43:04,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:06,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:08,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:09,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:43:10,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:10,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:43:10,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:43:10,883 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:43:12,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:14,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:14,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:43:16,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:43:16,715 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:43:16,715 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:43:16,717 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:43:16,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:16,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:43:17,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:18,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:20,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:43:20,759 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:43:20,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:22,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:24,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:25,688 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:43:25,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:43:25,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:43:25,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:43:26,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:28,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:30,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:31,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:43:32,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:34,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:36,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:43:36,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:38,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:40,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:40,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:43:40,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:43:40,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:43:41,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:43:42,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:44,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:46,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:47,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:43:48,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:50,761 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:43:50,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:52,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:43:52,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:54,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:43:55,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status -2024-04-25 20:43:55,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:43:55,883 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status -2024-04-25 20:43:56,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:01,810 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:44:01,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:44:06,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:44:10,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:44:16,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:44:17,146 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:44:17,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:44:17,146 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:44:17,149 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:44:17,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:17,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:44:18,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:20,765 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:44:20,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:21,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:44:22,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:44:22,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:24,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:25,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:44:26,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:28,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:44:28,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:30,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:44:30,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:32,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:33,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:44:34,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:35,919 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:44:36,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:38,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:38,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:44:40,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:44:40,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:42,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:44,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:44:44,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:45,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:44:46,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:47,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:44:47,055 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:44:47,055 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:44:47,056 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:44:47,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:47,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:44:48,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:49,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:44:50,767 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:44:50,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:44:50,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:52,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:54,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:55,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:44:55,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:44:56,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:44:58,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:00,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:45:00,931 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:45:00,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:02,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:04,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:05,931 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:45:06,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:45:06,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:08,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:10,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:45:10,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:11,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:45:12,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:14,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:15,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:45:16,831 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:45:16,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:18,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:20,769 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:45:20,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:45:20,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:22,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:45:22,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:24,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:25,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:45:26,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:27,664 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:45:28,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:30,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:45:31,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:32,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:45:32,129 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:45:32,130 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:45:32,133 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:45:33,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:33,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:45:33,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:45:34,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:35,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:35,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:45:37,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:38,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:45:39,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:40,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:45:41,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:43,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:43,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:45:45,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:45,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:45:47,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:49,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:49,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:45:50,771 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:45:50,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:45:51,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:53,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:54,788 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:45:55,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:55,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:45:57,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:45:59,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:00,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:46:00,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:46:01,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:03,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:05,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:05,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:46:05,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:46:07,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:09,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:10,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:46:10,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:46:11,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:13,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:15,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:15,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:46:16,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:46:17,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:17,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:46:17,218 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:46:17,218 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:46:17,220 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:46:18,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:46:19,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:20,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:20,773 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:46:20,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:46:21,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:21,812 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:46:23,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:25,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:25,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:46:27,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:27,233 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:46:29,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:31,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:46:31,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:32,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:46:33,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:35,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:36,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:46:37,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:38,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:46:39,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:40,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:46:41,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:43,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:43,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:46:45,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:46,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:46:47,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:48,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:46:49,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:50,776 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:46:51,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:46:51,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:53,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:54,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:46:55,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:55,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:46:57,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:59,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:46:59,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:47:01,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:47:01,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:02,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:47:02,261 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:47:02,262 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:47:02,263 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:47:03,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:03,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:47:04,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:05,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:47:05,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:06,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:47:07,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:09,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:10,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:47:10,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:47:11,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:13,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:15,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:15,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:47:16,007 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:47:17,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:19,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:20,777 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:47:21,007 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:47:21,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:21,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:47:23,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:25,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:25,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:47:27,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:27,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:47:29,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:31,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:47:31,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:32,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:47:33,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:35,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:36,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:47:37,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:38,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:47:39,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:40,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:47:41,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:43,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:43,687 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:47:45,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:46,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:47:47,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:47,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:47:47,315 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:47:47,316 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:47:47,318 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:47:48,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:47:49,085 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:47:49,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:50,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:50,780 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:47:51,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:47:51,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:53,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:54,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:47:55,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:55,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:47:57,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:59,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:47:59,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:48:01,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:48:01,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:03,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:05,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:05,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:48:06,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:48:07,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:09,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:10,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:48:10,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:48:11,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:13,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:15,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:16,015 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:48:16,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:48:17,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:19,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:20,793 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:48:21,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:48:21,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:21,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:48:23,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:25,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:25,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:48:26,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:48:27,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:29,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:31,043 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:48:31,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:32,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:48:32,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:48:32,405 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:48:32,406 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:48:32,407 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:48:33,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:33,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:48:34,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:35,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:36,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:48:37,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:37,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:48:39,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:40,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:48:41,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:43,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:48:43,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:45,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:46,111 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:48:47,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:48,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:48:49,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:50,785 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:48:51,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:48:51,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:53,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:54,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:48:55,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:55,977 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:48:57,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:59,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:48:59,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:49:01,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:49:01,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:03,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:04,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:49:05,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:06,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:49:07,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:09,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:10,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:49:10,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:49:11,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:13,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:15,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:15,741 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:49:16,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:49:17,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:17,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:49:17,579 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:49:17,579 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:49:17,581 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:49:18,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:18,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:49:20,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:20,788 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:49:20,789 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:49:21,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:49:22,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:24,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:25,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:49:26,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:26,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:49:28,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:30,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:31,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:49:31,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:49:32,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:34,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:36,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:49:36,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:37,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:49:38,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:40,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:40,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:49:42,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:42,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:49:44,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:46,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:49:46,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:48,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:49:48,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:50,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:50,789 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:49:51,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:49:52,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:53,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:49:54,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:55,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:49:56,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:58,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:49:59,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:50:00,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:01,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:50:02,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:02,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:50:02,698 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:50:02,698 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:50:02,700 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:50:03,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:50:04,470 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:50:04,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:06,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:50:06,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:08,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:09,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:50:10,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:10,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:50:12,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:14,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:15,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:50:16,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:50:16,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:18,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:20,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:20,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:50:20,803 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:50:21,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:50:22,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:24,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:25,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:50:26,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:50:26,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:28,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:30,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:31,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:50:31,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:50:32,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:34,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:36,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:50:36,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:36,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:50:38,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:40,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:40,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:50:42,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:50:42,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:44,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:46,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:50:46,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:47,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:50:47,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:50:47,701 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:50:47,701 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:50:47,703 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:50:48,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:48,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:50:50,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:50,793 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:50:51,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:50:52,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:53,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:50:54,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:55,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:50:56,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:50:58,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:50:58,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:00,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:01,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:51:02,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:03,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:51:04,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:06,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:51:06,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:08,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:09,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:51:10,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:10,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:51:12,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:14,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:14,736 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:51:16,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:51:16,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:18,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:20,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:51:20,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:20,796 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:51:21,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:51:22,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:24,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:25,522 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:51:25,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:51:26,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:28,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:30,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:30,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:51:31,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:51:32,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:32,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:51:32,765 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:51:32,766 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:51:32,767 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:51:33,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:51:34,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:36,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:51:36,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:51:36,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:38,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:40,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:40,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:51:41,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:51:42,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:44,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:46,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:51:46,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:47,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:51:48,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:50,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:50,798 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:51:51,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:51:52,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:51:52,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:54,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:55,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:51:56,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:51:58,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:51:58,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:00,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:01,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:52:02,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:03,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:52:04,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:06,537 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:52:06,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:08,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:08,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:52:10,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:10,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:52:12,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:14,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:52:14,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:16,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:52:16,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:17,898 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:52:17,900 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:52:17,901 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:52:17,902 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:52:18,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:18,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:52:19,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:52:20,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:20,811 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:52:21,555 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:52:22,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:24,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:25,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:52:25,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:52:26,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:28,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:30,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:52:30,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:31,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:52:32,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:34,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:35,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:52:36,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:52:36,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:38,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:40,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:40,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:52:41,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:52:42,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:44,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:46,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:52:46,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:52:46,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:48,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:50,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:50,803 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:52:51,573 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:52:51,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:52:52,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:54,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:55,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:52:56,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:52:57,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:52:58,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:00,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:01,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:53:02,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:02,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:53:02,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:53:02,970 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:53:02,971 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:53:02,972 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:53:03,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:03,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:53:04,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:06,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:53:06,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:08,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:53:08,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:10,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:10,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:53:12,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:13,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:53:14,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:16,629 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:53:16,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:18,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:19,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:53:20,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:20,805 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:53:21,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:53:22,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:24,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:53:24,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:25,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:53:26,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:28,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:29,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:53:30,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:31,632 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:53:32,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:34,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:35,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:53:36,639 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:53:36,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:38,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:40,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:40,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:53:40,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:53:42,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:44,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:46,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:53:46,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:53:46,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:48,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:53:48,096 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:53:48,096 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:53:48,098 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:53:48,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:48,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:53:49,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:50,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:50,807 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:53:51,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:53:51,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:53:52,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:54,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:55,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:53:56,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:53:57,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:53:58,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:00,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:01,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:54:02,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:54:02,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:04,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:06,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:54:06,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:07,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:54:08,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:10,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:10,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:54:12,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:13,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:54:14,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:16,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:54:16,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:18,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:54:18,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:20,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:20,825 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:54:21,655 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:54:22,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:24,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:54:24,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:25,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:54:26,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:28,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:29,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:54:30,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:31,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:54:32,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:33,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:54:33,127 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:54:33,128 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:54:33,129 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:54:33,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:33,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:54:34,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:34,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:54:36,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:54:36,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:38,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:40,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:54:40,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:40,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:54:42,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:44,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:45,682 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:54:46,658 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:54:46,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:48,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:50,811 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:54:50,812 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:54:50,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:51,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:54:52,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:54,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:55,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:54:56,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:54:56,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:54:58,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:00,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:01,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:55:01,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:55:02,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:04,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:06,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:55:06,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:07,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:55:08,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:10,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:10,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:55:12,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:55:12,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:14,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:16,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:55:16,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:18,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:55:18,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:55:18,072 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:55:18,073 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:55:18,074 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:55:18,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:18,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:55:19,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:20,813 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:55:20,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:21,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:55:22,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:23,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:55:24,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:25,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:55:26,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:28,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:55:28,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:30,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:31,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:55:32,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:34,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:55:34,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:36,762 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:55:36,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:38,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:39,662 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:55:40,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:40,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:55:42,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:44,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:45,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:55:46,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:55:46,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:48,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:50,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:55:50,816 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:55:50,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:51,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:55:52,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:54,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:55,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:55:55,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:55:56,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:55:58,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:00,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:01,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:56:01,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:56:02,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:03,168 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:56:03,170 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:56:03,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:56:03,172 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:56:03,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:03,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:56:04,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:06,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:56:06,819 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:56:06,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:08,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:10,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:10,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:56:12,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:56:12,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:14,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:16,836 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:56:16,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:17,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:56:18,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:20,831 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:56:20,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:21,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:56:22,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:22,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:56:24,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:25,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:56:26,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:28,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:56:28,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:30,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:31,838 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:56:32,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:33,725 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:56:34,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:36,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:56:36,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:38,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:39,138 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:56:40,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:41,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:56:42,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:44,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:56:44,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:46,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:56:46,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:48,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:56:48,201 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:56:48,201 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:56:48,203 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:56:48,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:48,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:56:49,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:49,973 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:56:50,821 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:56:50,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:51,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:56:52,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:54,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:55,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:56:56,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:56:56,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:56:58,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:00,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:57:00,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:01,865 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:57:02,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:04,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:06,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:57:06,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:57:06,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:08,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:10,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:11,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:57:11,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:57:12,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:14,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:16,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:57:16,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:57:16,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:18,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:20,823 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:57:20,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:21,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:57:22,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:57:22,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:24,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:26,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:57:26,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:27,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:57:29,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:31,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:31,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:57:33,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:33,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:57:33,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:57:33,135 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:57:33,136 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:57:33,137 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:57:34,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:34,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:57:35,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:36,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:57:37,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:38,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:57:39,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:41,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:41,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:57:43,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:43,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:57:45,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:46,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:57:47,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:49,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:49,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:57:50,825 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:57:51,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:51,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:57:53,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:54,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:57:55,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:56,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:57:57,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:57:59,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:00,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:58:01,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:01,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:58:03,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:05,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:05,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:58:06,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:58:07,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:09,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:10,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:58:11,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:58:11,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:13,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:15,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:16,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:58:16,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:58:17,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:18,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:58:18,198 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:58:18,198 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:58:18,200 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:58:19,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:19,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:58:20,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:20,838 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:58:21,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:21,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:58:21,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:58:23,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:25,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:26,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:58:27,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:27,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:58:29,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:31,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:31,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:58:32,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:58:33,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:35,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:36,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:58:37,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:37,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:58:39,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:41,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:58:41,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:43,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:43,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:58:45,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:46,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:58:47,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:48,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:58:49,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:50,829 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:58:51,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:51,898 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:58:53,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:54,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:58:55,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:56,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:58:57,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:59,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:58:59,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:59:01,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:01,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:59:03,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:03,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:59:03,263 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:59:03,263 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:59:03,265 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:59:04,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:04,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:59:05,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:59:05,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:06,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:59:07,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:09,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:10,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:59:11,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:59:11,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:13,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:15,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:15,835 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:59:16,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:59:17,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:19,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:20,831 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:59:21,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:59:21,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:21,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:59:23,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:25,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:26,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:59:26,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:59:27,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:29,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:31,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:31,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:59:32,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:59:33,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:35,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:36,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:59:37,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:37,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:59:39,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:41,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:59:41,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:42,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:59:43,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:45,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:46,948 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:59:47,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:48,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:59:48,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 20:59:48,268 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 20:59:48,268 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 20:59:48,270 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 20:59:48,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 20:59:49,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:50,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:50,834 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 20:59:51,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:51,949 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 20:59:53,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:53,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:59:55,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:56,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 20:59:57,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 20:59:59,030 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 20:59:59,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:01,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:01,950 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:00:03,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:04,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:00:05,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:06,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:00:07,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:09,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:09,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:00:11,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:00:11,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:13,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:15,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:00:15,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:16,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:00:17,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:19,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:20,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:00:20,846 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:00:21,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:21,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:00:23,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:25,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:26,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:00:26,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:00:27,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:29,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:31,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:31,450 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:00:31,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:00:33,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:00:33,287 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:00:33,288 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:00:33,289 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:00:33,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:33,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:00:34,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:35,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:36,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:00:36,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:00:37,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:39,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:41,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:00:41,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:42,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:00:43,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:45,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:46,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:00:47,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:47,664 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:00:49,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:50,838 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:00:51,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:51,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:00:53,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:00:53,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:55,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:56,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:00:57,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:00:58,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:00:59,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:01,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:01,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:01:03,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:03,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:01:05,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:06,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:01:07,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:09,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:01:09,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:11,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:01:11,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:13,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:14,706 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:01:15,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:16,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:01:17,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:18,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:01:18,352 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:01:18,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:01:18,355 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:01:18,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:01:19,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:20,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:01:20,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:20,840 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:01:21,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:22,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:01:23,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:25,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:25,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:01:26,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:01:27,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:29,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:30,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:01:31,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:32,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:01:33,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:35,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:36,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:01:37,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:01:37,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:39,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:41,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:01:41,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:41,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:01:43,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:45,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:47,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:01:47,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:01:47,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:49,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:50,843 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:01:51,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:52,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:01:52,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:01:53,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:55,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:56,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:01:57,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:01:57,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:01:59,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:01,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:02:01,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:03,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:02:03,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:02:03,365 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:02:03,366 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:02:03,367 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:02:03,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:03,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:02:05,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:06,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:02:06,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:07,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:08,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:02:09,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:11,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:02:11,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:13,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:14,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:02:15,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:16,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:02:17,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:19,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:19,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:02:20,863 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:02:21,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:02:21,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:23,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:24,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:02:25,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:26,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:02:27,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:29,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:30,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:02:31,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:02:31,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:33,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:35,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:35,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:02:36,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:02:37,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:39,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:41,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:02:41,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:02:41,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:43,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:45,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:46,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:02:46,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:02:47,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:48,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:02:48,313 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:02:48,313 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:02:48,315 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:02:48,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:02:49,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:50,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:50,847 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:02:51,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:02:51,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:51,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:02:53,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:55,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:56,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:02:57,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:02:57,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:02:59,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:01,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:03:01,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:02,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:03:03,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:05,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:06,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:03:07,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:08,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:03:09,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:11,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:03:11,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:13,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:03:13,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:15,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:16,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:03:17,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:18,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:03:19,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:20,850 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:03:21,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:03:21,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:23,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:24,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:03:25,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:26,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:03:27,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:29,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:29,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:03:31,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:03:31,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:33,363 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:03:33,364 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:03:33,365 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:03:33,367 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:03:33,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:33,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:03:35,138 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:03:35,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:36,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:03:36,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:37,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:39,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:40,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:03:41,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:03:41,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:43,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:45,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:45,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:03:46,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:03:47,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:49,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:50,852 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:03:51,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:03:51,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:03:51,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:53,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:55,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:56,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:03:56,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:03:57,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:03:59,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:01,065 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:04:01,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:02,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:04:03,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:05,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:06,065 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:04:07,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:04:07,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:09,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:11,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:04:11,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:12,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:04:13,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:15,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:16,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:04:17,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:18,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:04:18,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:04:18,372 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:04:18,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:04:18,374 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:04:18,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:04:19,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:20,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:20,864 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:04:21,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:04:21,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:23,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:23,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:04:25,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:26,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:04:27,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:29,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:04:29,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:31,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:04:31,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:33,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:34,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:04:35,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:36,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:04:37,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:39,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:39,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:04:41,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:04:41,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:43,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:45,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:04:45,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:46,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:04:47,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:49,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:50,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:04:50,857 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:04:51,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:04:51,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:53,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:55,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:56,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:04:56,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:04:57,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:04:59,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:01,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:05:01,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:05:01,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:03,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:05:03,377 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:05:03,377 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:05:03,379 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:05:03,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:03,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:05:05,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:06,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:05:06,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:06,932 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:05:07,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:09,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:11,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:05:11,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:12,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:05:13,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:15,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:16,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:05:17,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:17,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:05:19,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:20,859 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:05:21,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:05:21,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:23,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:05:23,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:25,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:26,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:05:27,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:28,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:05:29,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:31,085 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:05:31,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:33,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:33,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:05:35,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:36,125 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:05:37,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:39,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:05:39,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:41,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:05:41,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:43,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:44,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:05:45,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:46,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:05:47,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:48,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:05:48,311 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:05:48,311 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:05:48,313 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:05:48,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:05:49,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:50,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:05:50,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:50,861 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:05:51,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:05:51,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:53,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:55,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:05:55,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:56,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:05:56,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:05:58,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:00,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:00,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:06:01,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:06:02,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:04,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:06,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:06:06,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:06:06,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:08,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:10,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:11,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:06:11,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:06:12,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:14,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:16,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:06:16,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:17,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:06:18,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:20,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:20,873 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:06:21,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:06:22,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:06:22,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:24,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:26,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:06:26,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:27,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:06:28,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:30,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:31,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:06:32,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:33,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:06:33,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:06:33,192 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:06:33,192 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:06:33,195 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:06:33,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:06:34,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:36,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:06:36,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:38,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:06:38,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:40,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:41,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:06:42,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:43,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:06:44,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:46,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:06:46,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:48,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:49,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:06:50,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:50,866 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:06:51,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:06:52,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:54,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:06:54,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:56,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:06:56,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:06:58,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:00,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:07:00,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:01,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:07:02,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:04,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:05,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:07:06,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:07:06,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:08,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:10,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:10,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:07:11,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:07:12,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:14,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:16,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:07:16,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:07:16,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:18,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:07:18,175 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:07:18,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:07:18,177 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:07:18,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:18,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:07:19,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:20,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:20,868 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:07:21,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:07:21,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:07:22,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:24,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:26,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:07:26,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:27,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:07:28,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:30,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:31,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:07:32,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:07:32,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:34,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:36,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:07:36,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:37,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:07:38,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:40,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:41,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:07:42,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:43,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:07:44,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:46,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:07:46,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:48,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:07:48,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:50,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:50,870 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:07:51,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:07:52,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:54,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:07:54,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:56,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:07:56,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:58,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:07:59,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:08:00,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:01,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:08:02,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:03,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:08:03,206 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:08:03,207 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:08:03,208 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:08:03,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:03,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:08:04,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:04,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:08:06,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:08:06,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:08,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:10,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:08:10,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:11,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:08:12,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:14,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:15,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:08:16,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:08:16,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:18,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:20,888 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:08:20,889 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:08:20,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:21,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:08:22,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:24,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:26,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:08:26,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:08:26,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:28,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:30,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:31,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:08:31,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:08:32,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:34,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:36,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:08:36,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:37,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:08:38,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:40,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:41,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:08:42,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:08:42,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:44,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:46,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:08:46,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:48,156 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:08:48,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:08:48,192 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:08:48,193 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:08:48,194 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:08:48,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:48,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:08:50,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:50,875 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:08:51,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:51,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:08:53,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:53,568 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:08:55,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:56,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:08:57,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:08:58,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:08:59,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:01,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:01,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:09:03,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:04,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:09:05,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:06,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:09:07,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:09,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:09,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:09:11,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:09:11,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:13,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:15,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:15,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:09:16,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:09:17,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:19,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:20,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:09:20,877 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:09:21,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:21,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:09:23,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:25,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:25,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:09:26,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:09:27,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:29,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:31,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:31,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:09:31,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:09:33,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:09:33,199 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:09:33,199 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:09:33,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:33,201 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:09:34,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:34,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:09:35,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:36,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:09:36,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:09:37,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:39,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:41,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:09:41,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:42,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:09:43,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:45,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:46,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:09:47,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:47,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:09:49,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:50,879 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:09:51,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:51,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:09:52,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:09:53,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:55,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:56,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:09:57,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:09:58,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:09:59,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:01,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:01,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:10:03,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:03,748 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:10:05,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:06,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:10:07,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:09,168 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:10:09,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:11,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:10:11,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:13,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:14,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:10:15,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:16,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:10:17,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:18,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:10:18,190 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:10:18,190 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:10:18,192 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:10:18,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:10:19,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:19,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:10:20,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:20,892 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:10:21,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:21,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:10:23,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:25,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:25,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:10:26,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:10:27,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:29,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:30,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:10:31,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:31,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:10:33,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:35,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:36,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:10:36,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:10:37,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:39,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:41,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:10:41,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:41,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:10:43,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:45,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:46,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:10:46,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:10:47,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:49,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:50,884 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:10:51,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:51,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:10:52,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:10:53,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:55,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:56,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:10:57,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:10:57,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:10:59,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:01,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:01,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:11:03,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:11:03,107 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:11:03,109 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:11:03,109 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:11:03,111 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:11:03,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:03,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:11:04,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:05,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:06,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:11:07,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:08,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:11:09,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:11,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:11:11,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:13,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:13,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:11:15,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:16,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:11:17,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:19,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:11:19,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:20,886 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:11:21,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:21,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:11:23,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:24,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:11:25,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:26,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:11:27,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:29,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:30,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:11:31,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:31,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:11:33,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:35,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:35,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:11:36,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:11:37,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:39,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:40,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:11:41,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:11:41,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:43,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:45,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:46,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:11:46,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:11:47,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:48,109 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:11:48,111 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:11:48,111 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:11:48,113 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:11:48,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:11:49,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:50,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:50,887 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:11:51,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:51,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:11:51,674 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:11:53,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:55,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:56,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:11:57,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:11:57,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:11:59,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:01,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:01,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:12:02,449 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:12:03,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:05,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:06,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:12:07,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:07,836 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:12:09,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:11,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:12:11,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:13,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:12:13,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:15,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:16,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:12:17,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:18,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:12:19,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:20,901 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:12:21,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:12:21,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:23,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:24,056 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:12:25,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:26,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:12:27,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:29,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:29,469 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:12:31,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:12:31,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:33,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:12:33,101 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:12:33,102 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:12:33,103 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:12:33,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:33,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:12:34,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:34,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:12:35,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:36,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:12:37,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:39,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:40,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:12:41,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:12:41,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:43,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:45,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:45,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:12:46,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:12:47,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:49,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:50,892 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:12:50,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:12:51,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:12:51,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:53,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:55,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:56,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:12:56,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:12:57,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:12:59,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:01,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:13:01,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:01,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:13:03,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:05,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:06,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:13:07,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:13:07,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:09,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:11,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:13:11,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:12,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:13:13,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:15,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:16,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:13:17,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:18,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:13:18,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:13:18,074 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:13:18,075 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:13:18,076 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:13:18,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:13:19,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:20,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:20,895 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:13:21,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:13:21,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:23,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:13:23,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:25,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:26,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:13:27,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:28,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:13:29,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:31,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:13:31,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:33,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:34,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:13:35,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:36,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:13:37,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:39,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:39,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:13:41,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:13:41,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:43,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:45,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:13:45,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:46,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:13:47,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:49,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:50,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:13:50,897 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:13:51,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:13:51,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:53,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:55,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:55,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:13:56,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:13:57,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:13:59,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:01,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:14:01,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:14:01,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:03,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:14:03,080 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:14:03,080 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:14:03,082 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:14:03,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:03,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:14:04,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:05,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:06,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:14:06,649 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:14:07,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:09,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:11,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:14:11,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:12,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:14:13,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:15,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:16,407 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:14:17,450 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:14:17,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:19,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:20,924 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:14:21,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:14:21,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:22,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:14:23,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:25,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:26,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:14:27,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:28,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:14:29,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:31,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:14:31,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:33,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:33,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:14:35,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:36,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:14:37,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:39,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:14:39,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:41,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:14:41,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:43,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:44,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:14:45,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:46,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:14:47,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:48,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:14:48,085 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:14:48,086 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:14:48,087 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:14:48,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:14:49,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:49,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:14:50,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:50,901 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:14:51,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:14:51,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:53,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:55,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:14:55,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:56,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:14:57,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:14:59,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:00,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:15:01,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:15:01,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:03,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:05,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:06,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:15:06,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:15:07,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:09,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:11,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:15:11,407 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:15:11,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:13,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:15,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:16,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:15:16,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:15:17,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:19,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:20,903 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:15:21,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:15:21,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:21,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:15:23,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:25,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:26,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:15:27,588 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:15:27,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:29,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:31,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:15:31,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:32,987 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:15:33,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:15:33,024 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:15:33,025 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:15:33,026 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:15:33,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:33,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:15:34,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:35,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:36,466 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:15:37,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:38,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:15:39,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:41,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:15:41,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:43,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:43,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:15:45,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:46,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:15:47,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:49,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:15:49,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:50,906 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:15:51,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:15:51,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:53,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:54,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:15:55,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:56,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:15:57,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:59,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:15:59,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:16:01,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:16:01,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:03,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:05,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:16:05,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:06,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:16:07,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:09,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:10,746 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:16:11,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:16:11,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:13,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:15,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:16,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:16:16,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:16:17,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:17,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:16:17,961 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:16:17,961 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:16:17,963 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:16:18,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:16:19,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:20,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:20,918 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:16:21,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:16:21,526 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:16:21,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:23,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:25,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:26,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:16:26,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:16:27,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:29,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:31,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:16:31,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:32,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:16:33,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:35,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:36,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:16:37,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:16:37,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:39,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:41,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:16:41,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:43,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:16:43,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:45,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:46,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:16:47,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:48,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:16:49,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:50,910 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:16:51,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:16:51,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:53,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:53,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:16:55,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:56,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:16:57,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:16:59,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:16:59,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:01,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:17:01,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:02,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:17:02,927 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:17:02,927 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:17:02,930 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:17:03,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:03,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:17:04,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:17:04,497 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:17:04,499 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:17:04,499 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:17:04,501 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:17:04,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:04,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:17:05,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:06,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:07,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:08,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:17:09,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:10,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:17:11,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:17:11,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:15,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:17:16,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:17:19,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:20,912 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:17:20,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:17:21,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:21,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:17:23,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:25,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:26,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:17:26,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:17:27,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:29,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:31,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:31,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:17:31,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:17:33,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:35,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:36,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:17:37,369 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:17:37,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:39,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:41,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:17:41,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:42,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:17:43,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:45,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:46,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:17:47,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:48,192 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:17:49,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:50,915 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:17:51,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:51,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:17:53,597 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:17:53,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:55,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:56,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:17:57,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:17:58,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:17:59,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:01,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:01,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:18:03,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:18:03,399 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:18:03,399 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:18:03,401 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:18:03,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:03,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:18:04,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:18:04,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:05,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:06,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:18:07,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:09,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:18:09,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:11,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:18:11,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:13,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:15,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:18:15,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:16,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:18:17,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:19,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:20,567 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:18:20,918 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:18:21,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:21,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:18:23,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:25,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:25,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:18:26,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:18:27,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:29,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:31,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:18:31,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:31,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:18:33,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:35,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:36,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:18:36,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:18:37,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:39,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:41,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:18:41,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:42,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:18:43,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:45,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:46,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:18:47,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:18:47,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:48,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:18:48,381 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:18:48,382 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:18:48,383 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:18:48,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:18:49,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:50,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:50,920 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:18:51,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:18:51,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:52,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:18:53,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:55,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:56,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:18:57,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:18:58,361 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:18:59,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:01,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:19:01,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:03,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:19:03,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:05,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:06,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:19:06,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:08,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:09,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:19:10,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:11,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:19:12,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:14,555 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:19:15,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:16,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:19:17,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:19,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:19,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:19:20,923 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:19:21,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:21,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:19:23,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:25,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:25,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:19:26,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:19:27,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:29,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:30,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:19:31,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:31,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:19:33,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:33,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:19:33,402 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:19:33,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:19:33,404 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:19:34,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:34,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:19:35,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:36,168 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:19:36,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:19:37,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:39,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:41,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:41,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:19:41,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:19:43,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:45,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:46,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:19:46,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:19:47,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:49,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:50,925 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:19:51,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:51,889 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:19:52,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:19:53,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:55,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:56,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:19:57,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:19:57,835 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:19:59,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:01,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:01,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:20:03,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:03,228 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:20:05,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:06,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:20:07,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:08,622 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:20:09,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:11,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:11,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:20:13,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:14,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:20:15,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:16,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:20:17,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:18,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:20:18,477 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:20:18,478 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:20:18,480 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:20:19,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:19,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:20:19,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:20:20,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:20,927 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:20:21,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:21,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:20:23,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:24,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:20:25,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:26,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:20:27,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:29,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:30,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:20:31,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:31,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:20:33,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:35,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:35,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:20:36,938 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:20:37,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:39,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:40,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:20:41,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:20:41,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:43,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:45,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:46,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:20:46,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:20:47,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:49,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:50,929 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:20:51,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:51,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:20:51,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:20:53,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:55,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:56,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:20:57,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:20:57,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:20:59,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:01,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:01,941 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:21:02,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:21:03,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:03,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:21:03,431 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:21:03,432 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:21:03,433 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:21:04,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:04,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:21:05,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:06,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:21:07,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:07,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:21:09,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:11,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:21:11,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:13,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:13,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:21:15,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:16,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:21:17,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:18,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:21:19,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:20,931 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:21:21,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:21,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:21:23,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:24,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:21:25,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:26,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:21:27,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:29,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:29,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:21:31,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:31,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:21:33,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:34,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:21:35,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:36,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:21:37,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:39,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:40,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:21:41,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:21:41,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:43,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:45,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:45,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:21:47,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:21:47,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:48,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:21:48,333 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:21:48,334 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:21:48,335 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:21:49,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:49,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:21:50,976 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:21:50,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:21:51,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:52,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:21:53,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:55,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:56,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:21:56,511 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:21:57,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:21:59,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:01,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:01,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:22:02,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:22:03,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:05,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:07,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:22:07,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:07,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:22:09,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:11,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:22:11,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:12,664 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:22:13,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:15,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:17,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:22:17,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:18,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:22:19,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:20,979 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:22:21,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:22,014 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:22:23,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:23,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:22:25,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:26,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:22:27,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:28,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:22:29,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:31,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:32,015 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:22:33,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:22:33,273 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:22:33,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:22:33,276 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:22:33,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:33,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:22:34,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:22:35,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:37,015 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:22:37,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:39,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:39,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:22:41,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:22:41,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:43,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:45,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:22:45,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:47,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:22:47,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:49,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:50,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:22:50,981 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:22:51,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:52,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:22:53,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:55,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:55,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:22:56,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:22:57,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:22:59,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:01,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:23:01,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:02,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:23:03,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:05,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:06,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:23:07,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:23:07,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:09,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:11,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:23:11,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:12,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:23:13,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:15,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:17,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:23:17,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:17,470 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:23:18,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:23:18,298 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:23:18,298 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:23:18,300 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:23:18,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:23:19,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:20,983 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:23:21,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:22,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:23:22,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:23:23,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:25,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:26,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:23:27,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:28,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:23:29,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:31,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:32,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:23:33,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:33,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:23:35,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:37,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:23:37,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:39,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:23:39,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:41,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:23:41,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:43,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:44,407 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:23:45,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:47,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:23:47,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:49,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:49,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:23:50,985 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:23:51,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:52,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:23:53,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:55,139 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:23:55,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:56,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:23:57,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:23:59,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:00,519 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:24:01,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:02,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:24:03,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:24:03,135 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:24:03,135 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:24:03,137 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:24:03,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:03,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:24:05,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:05,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:24:07,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:24:07,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:09,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:11,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:24:11,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:24:11,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:13,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:15,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:16,694 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:24:17,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:24:17,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:19,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:20,987 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:24:21,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:22,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:24:22,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:24:23,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:25,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:26,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:24:27,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:27,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:24:29,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:31,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:32,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:24:32,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:24:33,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:35,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:37,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:24:37,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:38,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:24:39,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:41,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:24:41,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:43,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:43,789 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:24:45,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:47,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:24:47,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:48,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:24:48,203 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:24:48,203 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:24:48,205 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:24:48,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:24:49,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:24:49,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:50,989 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:24:51,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:52,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:24:53,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:54,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:24:55,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:56,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:24:57,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:59,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:24:59,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:25:01,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:02,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:25:03,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:05,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:25:05,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:07,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:25:07,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:09,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:10,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:25:11,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:25:11,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:13,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:15,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:16,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:25:17,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:25:17,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:19,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:20,991 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:25:21,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:21,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:25:22,096 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:25:23,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:25,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:26,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:25:27,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:27,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:25:29,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:31,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:32,097 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:25:33,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:25:33,143 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:25:33,144 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:25:33,145 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:25:33,147 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:25:33,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:33,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:25:35,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:37,098 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:25:37,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:38,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:25:39,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:41,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:25:41,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:43,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:43,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:25:45,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:47,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:25:47,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:49,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:25:49,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:50,993 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:25:51,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:52,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:25:53,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:54,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:25:55,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:56,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:25:57,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:25:59,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:00,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:26:01,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:02,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:26:03,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:05,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:26:05,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:07,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:26:07,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:09,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:10,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:26:11,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:26:11,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:13,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:15,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:16,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:26:17,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:26:17,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:18,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:26:18,086 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:26:18,086 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:26:18,088 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:26:18,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:26:19,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:21,026 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:26:21,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:21,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:26:22,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:26:23,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:25,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:26,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:26:27,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:26:27,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:29,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:31,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:32,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:26:32,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:26:33,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:35,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:37,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:26:37,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:37,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:26:39,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:41,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:26:41,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:43,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:26:43,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:45,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:46,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:26:47,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:48,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:26:49,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:50,997 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:26:51,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:26:51,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:53,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:54,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:26:55,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:56,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:26:57,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:26:59,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:26:59,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:01,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:27:01,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:03,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:27:03,088 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:27:03,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:27:03,091 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:27:03,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:03,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:27:04,852 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:27:05,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:06,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:27:07,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:09,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:10,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:27:11,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:27:11,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:13,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:15,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:15,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:27:16,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:27:17,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:19,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:20,999 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:27:21,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:27:21,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:27:21,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:23,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:25,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:26,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:27:26,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:27:27,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:29,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:31,186 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:27:31,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:31,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:27:33,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:35,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:36,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:27:37,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:27:37,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:39,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:41,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:27:41,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:42,663 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:27:43,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:45,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:46,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:27:47,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:48,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:27:48,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:27:48,103 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:27:48,103 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:27:48,106 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:27:48,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:27:49,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:50,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:51,002 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:27:51,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:27:51,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:53,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:27:53,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:55,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:56,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:27:57,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:27:58,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:27:59,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:01,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:28:01,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:03,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:04,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:28:05,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:06,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:28:07,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:09,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:28:09,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:11,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:28:11,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:13,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:15,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:28:15,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:16,192 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:28:17,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:19,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:20,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:28:21,025 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:28:21,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:28:21,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:23,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:25,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:25,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:28:26,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:28:27,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:29,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:31,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:28:31,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:28:31,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:33,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:28:33,062 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:28:33,062 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:28:33,064 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:28:33,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:33,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:28:34,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:35,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:36,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:28:36,694 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:28:37,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:39,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:41,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:28:41,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:42,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:28:43,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:45,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:46,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:28:47,471 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:28:47,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:49,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:51,007 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:28:51,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:28:51,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:52,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:28:53,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:55,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:56,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:28:57,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:28:58,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:28:59,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:01,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:29:01,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:03,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:29:03,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:05,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:06,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:29:07,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:09,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:29:09,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:11,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:29:11,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:13,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:14,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:29:15,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:16,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:29:17,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:18,165 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:29:18,167 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:29:18,167 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:29:18,168 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:29:18,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:29:19,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:19,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:29:20,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:21,009 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:29:21,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:29:21,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:23,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:25,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:29:25,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:26,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:29:27,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:29,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:30,706 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:29:31,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:29:31,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:33,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:35,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:36,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:29:36,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:29:37,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:39,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:41,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:29:41,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:29:41,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:43,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:46,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:46,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:29:46,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:29:48,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:50,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:51,011 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:29:51,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:29:52,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:29:52,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:54,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:56,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:29:56,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:29:57,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:29:58,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:00,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:01,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:30:02,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:03,056 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:30:03,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:30:03,089 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:30:03,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:30:03,092 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:30:04,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:04,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:30:06,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:06,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:30:08,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:08,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:30:10,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:11,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:30:12,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:13,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:30:14,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:16,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:16,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:30:18,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:19,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:30:20,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:21,023 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:30:21,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:30:22,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:24,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:24,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:30:26,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:26,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:30:28,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:30,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:30:30,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:31,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:30:32,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:34,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:35,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:30:36,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:36,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:30:38,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:40,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:40,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:30:41,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:30:42,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:44,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:46,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:46,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:30:46,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:30:47,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:30:47,989 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:30:47,990 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:30:47,992 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:30:48,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:48,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:30:50,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:51,015 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:30:51,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:30:51,559 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:30:52,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:54,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:56,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:30:56,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:30:56,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:30:58,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:00,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:01,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:31:02,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:02,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:31:04,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:06,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:06,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:31:07,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:31:08,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:10,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:11,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:31:12,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:13,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:31:14,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:16,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:16,256 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:31:18,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:18,526 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:31:20,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:21,017 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:31:21,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:31:22,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:23,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:31:24,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:26,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:26,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:31:28,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:29,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:31:30,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:31,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:31:32,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:32,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:31:32,892 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:31:32,892 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:31:32,894 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:31:33,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:31:34,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:34,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:31:36,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:36,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:31:38,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:40,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:31:40,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:41,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:31:42,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:44,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:45,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:31:46,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:46,260 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:31:48,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:50,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:50,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:31:51,020 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:31:51,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:31:52,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:54,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:56,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:31:56,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:31:56,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:31:58,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:00,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:01,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:32:01,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:32:02,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:04,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:06,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:06,320 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:32:07,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:32:08,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:10,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:11,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:32:12,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:12,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:32:14,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:16,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:16,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:32:17,782 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:32:17,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:32:17,816 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:32:17,816 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:32:17,818 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:32:18,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:18,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:32:20,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:21,022 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:32:21,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:32:22,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:23,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:32:24,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:26,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:32:26,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:28,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:28,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:32:30,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:31,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:32:32,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:33,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:32:34,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:36,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:36,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:32:37,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:39,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:39,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:32:41,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:32:41,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:43,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:44,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:32:45,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:46,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:32:47,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:49,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:50,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:32:51,024 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:32:51,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:51,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:32:53,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:55,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:55,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:32:56,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:32:57,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:32:59,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:00,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:33:01,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:01,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:33:02,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:33:02,699 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:33:02,700 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:33:02,701 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:33:03,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:03,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:33:04,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:05,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:06,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:33:06,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:33:07,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:09,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:11,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:33:11,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:11,635 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:33:13,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:15,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:16,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:33:17,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:33:17,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:19,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:21,026 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:33:21,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:21,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:33:22,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:33:23,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:25,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:26,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:33:27,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:27,795 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:33:29,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:31,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:31,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:33:33,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:33:33,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:35,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:36,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:33:37,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:38,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:33:39,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:41,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:33:41,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:43,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:43,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:33:45,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:46,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:33:47,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:47,538 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:33:47,540 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:33:47,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:33:47,542 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:33:48,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:48,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:33:49,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:49,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:33:51,029 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:33:51,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:51,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:33:53,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:54,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:33:55,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:56,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:33:57,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:33:59,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:00,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:34:01,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:01,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:34:03,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:05,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:05,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:34:06,422 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:34:07,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:09,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:10,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:34:11,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:34:11,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:13,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:15,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:16,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:34:16,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:34:17,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:19,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:21,031 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:34:21,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:21,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:34:21,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:34:23,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:25,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:26,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:34:27,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:34:27,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:29,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:31,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:31,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:34:32,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:34:32,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:34:32,490 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:34:32,491 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:34:32,492 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:34:33,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:33,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:34:34,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:35,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:36,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:34:37,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:37,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:34:39,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:41,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:34:41,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:43,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:34:43,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:45,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:46,427 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:34:47,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:48,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:34:49,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:51,033 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:34:51,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:51,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:34:53,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:53,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:34:55,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:56,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:34:57,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:59,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:34:59,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:35:01,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:01,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:35:03,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:04,788 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:35:05,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:06,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:35:07,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:09,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:10,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:35:11,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:35:11,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:13,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:15,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:15,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:35:16,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:35:17,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:35:17,391 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:35:17,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:35:17,394 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:35:17,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:17,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:35:18,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:19,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:20,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:35:21,035 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:35:21,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:21,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:35:23,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:25,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:26,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:35:26,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:35:27,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:29,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:31,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:31,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:35:31,727 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:35:33,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:35,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:36,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:35:37,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:35:37,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:39,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:41,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:35:41,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:42,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:35:43,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:45,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:46,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:35:47,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:47,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:35:49,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:51,037 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:35:51,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:51,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:35:53,338 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:35:53,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:55,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:56,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:35:57,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:35:58,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:35:59,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:01,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:01,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:36:02,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:36:02,336 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:36:02,336 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:36:02,338 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:36:02,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:36:03,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:04,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:36:04,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:05,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:06,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:36:07,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:09,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:09,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:36:11,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:36:11,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:13,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:14,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:36:15,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:16,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:36:17,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:19,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:20,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:36:21,040 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:36:21,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:21,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:36:23,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:25,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:25,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:36:26,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:36:27,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:29,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:30,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:36:31,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:31,501 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:36:33,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:35,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:36,361 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:36:36,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:36:37,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:39,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:41,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:36:41,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:41,762 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:36:43,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:45,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:46,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:36:47,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:36:47,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:36:47,205 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:36:47,205 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:36:47,207 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:36:47,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:47,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:36:48,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:49,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:51,041 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:36:51,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:36:51,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:52,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:36:53,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:55,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:56,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:36:57,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:36:57,984 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:36:59,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:01,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:37:01,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:03,369 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:37:03,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:05,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:06,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:37:07,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:08,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:37:09,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:11,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:37:11,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:13,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:14,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:37:15,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:16,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:37:17,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:19,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:19,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:37:21,043 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:37:21,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:37:21,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:23,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:24,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:37:25,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:26,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:37:27,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:29,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:30,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:37:31,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:37:31,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:32,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:37:32,198 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:37:32,198 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:37:32,200 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:37:32,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:37:33,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:34,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:35,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:35,760 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:37:36,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:37:37,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:39,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:41,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:37:41,238 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:37:41,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:43,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:45,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:46,511 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:37:46,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:37:47,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:49,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:51,046 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:37:51,512 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:37:51,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:51,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:37:53,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:55,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:56,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:37:57,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:37:57,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:37:59,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:01,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:38:01,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:02,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:38:03,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:05,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:06,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:38:07,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:08,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:38:09,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:11,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:38:11,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:13,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:38:13,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:15,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:16,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:38:17,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:38:17,092 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:38:17,093 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:38:17,095 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:38:17,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:17,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:38:18,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:18,851 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:38:19,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:21,060 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:38:21,551 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:38:21,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:23,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:24,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:38:25,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:26,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:38:27,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:29,606 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:38:29,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:31,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:38:31,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:33,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:34,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:38:35,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:36,553 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:38:37,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:39,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:40,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:38:41,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:38:41,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:43,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:45,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:45,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:38:46,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:38:47,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:49,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:51,050 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:38:51,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:38:51,555 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:38:51,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:53,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:55,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:56,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:38:56,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:38:57,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:38:59,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:01,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:39:01,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:01,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:39:01,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:39:01,945 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:39:01,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:39:01,947 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:39:02,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:39:03,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:04,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:05,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:06,573 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:39:07,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:39:07,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:09,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:11,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:39:11,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:12,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:39:13,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:15,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:16,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:39:17,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:18,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:39:19,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:21,052 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:39:21,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:39:21,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:23,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:39:23,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:25,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:26,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:39:27,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:28,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:39:29,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:31,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:39:31,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:33,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:34,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:39:35,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:36,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:39:37,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:39,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:39:39,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:41,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:39:41,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:43,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:45,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:39:45,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:46,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:39:46,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:39:46,857 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:39:46,857 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:39:46,859 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:39:46,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:39:47,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:48,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:49,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:50,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:39:51,055 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:39:51,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:39:51,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:53,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:55,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:39:55,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:39:56,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:39:57,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:00,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:01,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:40:01,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:40:02,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:04,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:06,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:06,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:40:06,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:40:08,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:10,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:11,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:40:11,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:40:12,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:14,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:16,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:16,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:40:17,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:40:18,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:20,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:21,067 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:40:21,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:40:22,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:22,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:40:24,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:26,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:26,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:40:28,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:28,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:40:30,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:31,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:40:31,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:40:31,725 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:40:31,725 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:40:31,728 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:40:32,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:32,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:40:33,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:33,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:40:34,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:36,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:36,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:40:38,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:38,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:40:40,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:41,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:40:42,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:44,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:44,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:40:46,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:46,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:40:48,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:49,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:40:50,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:51,059 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:40:51,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:40:52,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:54,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:54,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:40:56,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:40:56,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:40:58,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:00,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:00,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:41:01,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:41:02,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:04,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:05,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:41:06,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:06,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:41:08,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:10,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:11,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:41:11,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:41:12,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:14,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:16,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:16,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:41:16,605 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:41:16,607 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:41:16,607 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:41:16,610 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:41:16,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:41:17,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:41:18,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:19,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:20,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:21,061 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:41:21,674 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:41:21,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:41:22,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:24,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:26,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:26,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:41:27,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:41:28,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:30,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:31,675 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:41:32,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:32,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:41:34,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:36,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:36,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:41:38,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:41:38,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:40,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:41,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:41:42,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:43,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:41:44,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:46,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:46,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:41:48,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:48,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:41:50,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:51,063 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:41:51,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:41:52,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:54,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:54,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:41:56,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:56,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:41:58,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:41:59,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:42:00,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:01,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:42:01,498 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:42:01,498 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:42:01,501 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:42:01,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:42:02,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:02,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:42:03,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:04,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:05,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:42:06,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:06,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:42:08,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:10,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:10,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:42:11,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:42:12,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:14,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:15,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:42:16,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:16,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:42:18,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:20,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:21,075 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:42:21,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:42:21,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:42:22,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:24,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:26,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:26,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:42:26,586 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:42:28,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:30,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:31,711 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:42:31,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:42:32,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:34,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:36,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:36,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:42:37,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:42:38,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:40,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:41,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:42:42,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:42,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:42:44,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:46,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:46,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:42:46,336 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:42:46,336 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:42:46,338 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:42:46,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:42:47,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:42:48,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:42:48,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:50,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:51,068 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:42:51,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:42:52,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:53,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:42:54,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:56,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:56,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:42:58,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:42:58,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:43:00,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:01,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:43:02,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:04,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:43:04,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:06,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:06,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:43:08,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:09,582 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:43:10,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:11,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:43:12,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:14,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:14,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:43:16,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:16,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:43:18,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:20,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:20,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:43:21,070 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:43:21,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:43:22,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:24,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:25,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:43:26,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:43:26,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:28,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:30,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:31,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:43:31,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:43:31,248 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:43:31,248 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:43:31,250 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:43:31,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:43:31,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:43:32,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:34,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:36,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:36,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:43:36,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:43:38,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:40,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:41,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:43:42,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:43:42,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:44,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:46,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:46,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:43:47,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:43:48,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:50,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:51,072 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:43:51,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:43:52,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:52,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:43:54,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:56,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:43:56,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:43:58,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:43:58,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:00,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:01,760 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:44:02,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:03,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:44:04,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:06,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:06,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:44:08,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:08,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:44:10,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:11,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:44:12,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:14,275 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:44:14,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:16,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:44:16,102 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:44:16,102 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:44:16,104 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:44:16,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:16,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:44:16,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:44:18,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:19,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:44:20,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:21,085 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:44:21,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:44:22,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:24,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:25,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:44:26,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:44:26,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:28,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:30,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:30,429 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:44:31,782 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:44:32,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:34,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:35,836 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:44:36,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:36,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:44:38,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:40,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:41,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:44:41,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:44:42,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:44,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:46,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:46,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:44:46,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:44:48,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:50,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:51,077 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:44:51,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:44:51,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:44:52,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:54,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:56,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:44:56,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:44:57,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:44:58,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:00,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:00,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:45:00,973 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:45:00,973 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:45:00,976 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:45:01,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:45:01,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:45:02,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:02,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:45:04,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:06,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:06,810 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:45:08,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:45:08,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:10,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:11,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:45:12,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:13,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:45:14,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:16,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:16,811 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:45:18,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:18,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:45:20,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:21,078 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:45:21,812 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:45:22,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:24,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:45:24,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:26,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:45:26,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:28,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:29,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:45:30,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:31,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:45:32,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:34,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:34,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:45:36,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:36,838 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:45:38,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:40,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:45:40,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:41,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:45:42,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:44,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:45,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:45:45,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:45:45,691 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:45:45,691 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:45:45,694 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:45:46,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:46,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:45:46,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:45:48,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:50,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:51,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:45:51,081 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:45:51,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:45:52,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:54,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:56,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:45:56,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:45:56,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:45:58,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:00,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:01,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:46:01,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:46:02,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:04,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:06,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:06,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:46:07,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:46:08,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:10,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:11,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:46:12,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:12,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:46:14,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:16,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:16,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:46:17,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:46:18,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:20,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:21,097 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:46:21,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:46:22,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:23,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:46:24,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:26,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:46:26,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:28,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:28,730 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:46:30,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:30,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:46:30,559 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:46:30,559 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:46:30,561 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:46:31,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:46:31,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:46:32,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:33,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:34,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:46:35,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:36,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:46:37,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:39,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:46:39,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:41,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:46:41,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:43,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:44,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:46:45,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:46,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:46:47,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:49,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:50,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:46:51,085 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:46:51,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:51,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:46:53,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:55,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:55,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:46:56,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:46:57,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:46:59,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:01,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:47:01,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:01,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:47:03,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:05,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:06,629 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:47:06,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:47:07,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:09,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:11,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:47:11,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:12,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:47:13,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:15,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:15,618 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:47:15,620 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:47:15,620 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:47:15,622 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:47:16,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:16,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:47:16,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:47:17,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:47:17,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:19,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:21,087 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:47:21,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:21,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:47:22,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:47:23,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:25,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:26,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:47:27,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:28,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:47:29,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:31,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:31,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:47:33,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:47:33,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:35,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:36,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:47:37,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:38,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:47:39,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:41,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:47:41,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:43,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:44,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:47:45,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:46,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:47:47,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:49,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:49,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:47:51,089 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:47:51,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:51,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:47:53,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:55,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:47:55,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:56,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:47:57,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:47:59,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:00,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:48:00,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:48:00,497 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:48:00,497 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:48:00,499 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:48:00,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:48:01,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:01,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:48:02,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:03,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:05,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:05,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:48:06,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:48:07,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:09,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:11,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:48:11,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:48:11,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:13,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:15,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:16,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:48:16,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:48:17,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:19,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:21,094 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:48:21,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:21,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:48:22,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:48:23,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:25,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:26,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:48:27,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:48:27,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:29,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:31,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:31,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:48:32,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:48:33,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:35,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:36,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:48:37,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:38,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:48:39,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:41,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:48:41,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:43,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:48:43,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:45,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:48:45,411 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:48:45,412 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:48:45,414 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:48:45,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:45,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:48:46,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:46,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:48:47,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:48,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:48:49,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:51,096 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:48:51,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:51,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:48:53,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:54,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:48:55,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:56,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:48:57,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:48:59,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:48:59,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:01,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:01,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:49:03,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:05,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:49:05,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:06,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:49:07,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:09,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:10,449 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:49:11,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:49:11,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:13,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:15,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:49:15,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:17,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:49:17,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:19,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:21,098 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:49:21,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:49:21,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:22,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:49:23,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:25,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:26,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:49:26,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:49:27,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:29,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:30,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:49:30,238 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:49:30,238 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:49:30,240 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:49:30,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:49:31,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:32,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:49:32,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:49:32,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:33,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:35,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:37,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:49:37,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:49:37,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:39,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:41,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:49:41,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:42,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:49:43,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:45,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:47,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:49:47,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:48,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:49:49,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:51,100 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:49:51,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:52,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:49:53,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:49:53,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:55,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:56,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:49:58,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:49:58,852 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:50:00,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:02,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:02,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:50:04,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:04,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:50:06,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:07,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:50:08,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:09,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:50:10,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:11,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:50:12,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:14,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:14,949 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:50:14,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:50:14,984 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:50:14,985 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:50:14,986 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:50:15,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:50:16,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:17,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:17,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:50:18,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:20,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:20,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:50:21,113 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:50:22,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:22,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:50:24,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:25,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:50:26,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:26,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:50:28,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:30,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:31,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:50:32,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:32,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:50:34,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:36,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:36,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:50:37,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:50:38,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:40,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:41,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:50:41,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:50:42,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:44,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:46,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:47,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:50:47,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:50:48,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:50,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:51,105 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:50:52,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:52,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:50:52,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:50:54,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:56,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:56,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:50:58,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:50:58,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:50:59,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:50:59,875 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:50:59,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:50:59,877 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:51:00,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:00,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:51:02,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:02,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:51:03,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:03,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:51:04,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:06,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:07,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:51:08,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:08,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:51:10,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:11,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:51:12,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:14,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:14,185 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:51:16,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:17,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:51:18,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:19,573 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:51:20,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:21,107 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:51:22,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:22,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:51:24,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:24,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:51:26,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:26,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:51:28,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:30,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:30,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:51:32,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:32,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:51:34,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:35,726 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:51:36,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:37,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:51:38,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:40,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:41,098 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:51:41,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:51:42,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:44,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:44,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:51:44,705 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:51:44,706 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:51:44,708 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:51:45,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:51:46,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:46,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:51:47,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:47,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:51:48,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:50,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:51,109 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:51:51,831 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:51:52,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:52,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:51:54,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:56,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:51:56,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:51:57,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:51:58,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:00,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:02,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:02,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:52:02,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:52:04,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:06,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:07,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:52:07,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:52:08,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:10,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:11,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:52:12,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:13,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:52:14,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:16,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:17,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:52:18,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:18,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:52:20,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:21,128 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:52:22,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:22,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:52:24,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:52:24,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:26,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:26,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:52:28,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:29,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:52:29,521 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:52:29,523 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:52:29,523 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:52:29,525 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:52:30,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:30,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:52:31,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:32,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:32,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:52:34,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:34,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:52:36,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:37,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:52:38,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:40,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:40,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:52:41,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:52:42,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:44,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:45,606 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:52:46,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:47,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:52:48,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:50,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:50,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:52:51,113 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:52:52,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:52,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:52:54,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:56,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:52:56,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:52:56,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:52:58,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:00,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:01,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:53:02,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:53:02,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:04,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:06,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:06,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:53:07,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:53:08,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:10,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:11,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:53:12,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:53:12,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:13,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:53:13,993 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:53:13,994 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:53:13,996 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:53:14,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:14,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:53:16,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:17,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:53:17,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:17,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:53:18,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:20,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:21,115 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:53:22,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:53:22,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:22,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:53:24,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:26,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:26,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:53:28,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:53:28,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:30,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:32,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:53:32,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:33,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:53:34,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:36,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:37,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:53:38,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:38,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:53:40,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:41,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:53:42,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:44,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:53:44,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:46,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:47,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:53:48,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:49,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:53:50,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:51,118 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:53:52,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:53:52,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:54,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:54,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:53:56,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:53:56,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:58,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:53:58,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:53:58,478 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:53:58,479 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:53:58,481 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:53:59,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:54:00,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:54:00,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:02,228 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:54:02,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:04,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:05,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:54:06,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:07,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:54:08,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:10,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:10,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:54:11,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:54:12,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:14,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:16,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:54:16,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:17,239 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:54:18,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:20,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:21,130 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:54:21,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:54:22,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:54:22,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:24,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:26,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:54:26,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:26,948 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:54:28,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:30,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:32,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:54:32,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:54:32,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:34,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:36,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:37,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:54:37,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:54:38,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:40,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:41,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:54:42,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:42,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:54:42,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:54:42,995 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:54:42,995 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:54:42,998 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:54:43,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:54:44,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:46,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:47,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:54:48,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:54:48,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:50,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:51,122 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:54:52,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:54:52,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:53,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:54:54,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:56,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:54:56,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:58,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:54:59,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:55:00,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:02,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:55:02,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:04,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:55:04,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:06,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:07,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:55:08,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:09,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:55:10,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:11,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:55:12,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:14,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:15,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:55:16,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:55:16,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:18,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:20,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:55:20,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:21,124 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:55:21,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:55:22,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:24,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:25,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:55:26,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:55:26,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:27,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:55:27,487 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:55:27,487 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:55:27,489 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:55:28,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:28,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:55:30,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:30,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:55:31,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:55:32,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:34,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:36,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:55:36,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:55:36,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:38,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:40,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:41,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:55:41,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:55:42,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:44,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:46,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:55:46,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:46,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:55:48,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:50,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:51,126 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:55:51,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:55:52,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:55:52,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:54,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:56,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:55:56,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:55:57,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:55:58,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:00,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:01,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:56:02,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:03,014 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:56:04,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:06,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:56:06,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:08,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:56:08,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:10,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:11,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:56:11,938 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:56:11,939 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:56:11,939 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:56:11,942 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:56:12,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:12,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:56:13,674 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:56:14,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:16,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:56:16,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:18,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:18,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:56:20,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:21,140 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:56:21,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:56:22,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:24,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:56:24,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:26,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:56:26,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:28,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:29,631 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:56:30,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:31,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:56:32,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:34,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:34,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:56:36,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:56:36,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:38,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:40,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:56:40,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:41,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:56:42,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:44,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:45,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:56:46,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:56:46,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:48,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:50,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:50,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:56:51,131 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:56:51,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:56:52,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:54,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:56,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:56:56,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:56:56,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:56:56,391 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:56:56,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:56:56,394 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:56:56,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:56:56,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:56:58,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:00,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:01,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:57:01,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:57:02,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:04,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:06,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:57:06,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:07,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:57:08,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:10,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:11,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:57:12,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:57:12,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:14,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:16,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:57:16,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:17,663 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:57:18,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:20,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:21,133 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:57:21,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:57:22,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:22,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:57:24,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:26,256 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:57:26,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:28,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:57:28,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:30,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:31,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:57:32,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:33,622 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:57:34,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:36,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:57:36,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:38,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:38,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:57:40,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:40,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:57:40,788 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:57:40,789 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:57:40,791 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:57:41,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:57:41,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:57:42,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:44,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:57:44,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:46,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:57:46,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:48,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:49,692 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:57:50,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:51,135 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:57:51,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:57:52,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:54,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:55,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:57:56,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:57:56,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:57:58,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:00,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:58:00,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:01,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:58:02,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:04,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:05,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:58:06,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:58:06,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:08,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:10,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:11,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:58:11,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:58:12,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:14,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:16,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:58:16,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:58:16,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:18,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:20,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:21,148 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:58:21,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:58:21,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:58:22,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:24,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:25,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:58:25,284 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:58:25,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:58:25,287 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:58:25,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:58:26,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:58:26,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:27,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:58:28,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:30,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:31,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:58:32,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:58:32,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:34,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:36,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:58:36,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:37,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:58:38,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:40,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:41,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:58:42,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:42,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:58:43,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:45,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:46,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:58:47,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:48,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:58:49,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:51,139 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:58:51,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:58:51,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:53,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:53,810 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:58:55,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:56,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:58:57,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:58:59,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:58:59,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:01,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:59:01,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:03,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:04,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:59:05,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:06,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:59:07,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:09,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:09,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:59:09,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:59:09,893 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:59:09,893 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:59:09,897 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:59:10,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:10,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:59:11,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:59:11,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:13,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:15,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:59:15,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:16,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:59:17,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:19,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:20,517 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:59:21,142 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:59:21,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:59:21,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:23,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:25,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:59:25,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:26,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:59:27,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:29,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:31,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:59:31,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:59:31,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:33,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:35,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:36,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:59:36,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:59:37,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:39,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:41,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:59:41,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:59:41,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:43,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:46,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:46,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:59:47,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:59:48,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:50,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:51,144 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 21:59:51,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 21:59:52,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:52,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:59:54,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:54,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 21:59:54,383 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 21:59:54,384 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 21:59:54,386 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 21:59:55,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 21:59:56,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:56,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 21:59:57,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 21:59:57,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 21:59:58,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:00,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:01,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:00:02,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:03,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:00:04,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:06,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:06,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:00:08,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:08,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:00:10,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:11,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:00:12,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:13,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:00:14,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:16,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:16,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:00:18,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:19,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:00:20,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:21,146 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:00:21,427 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:00:22,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:24,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:24,568 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:00:26,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:26,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:00:28,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:29,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:00:30,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:31,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:00:32,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:34,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:35,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:00:36,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:36,429 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:00:38,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:38,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:00:38,791 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:00:38,792 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:00:38,793 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:00:39,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:00:40,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:40,537 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:00:41,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:41,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:00:42,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:44,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:45,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:00:46,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:46,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:00:48,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:50,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:51,148 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:00:51,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:00:51,431 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:00:52,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:54,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:56,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:00:56,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:00:56,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:00:58,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:00,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:01,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:01:01,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:01:02,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:04,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:06,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:06,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:01:07,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:01:08,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:10,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:11,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:01:12,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:12,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:01:14,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:16,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:16,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:01:17,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:01:18,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:20,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:21,151 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:01:21,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:01:22,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:23,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:01:23,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:01:23,312 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:01:23,313 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:01:23,314 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:01:24,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:24,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:01:25,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:26,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:26,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:01:28,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:28,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:01:30,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:31,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:01:32,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:33,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:01:34,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:36,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:36,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:01:38,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:39,255 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:01:40,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:41,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:01:42,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:44,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:44,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:01:46,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:46,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:01:48,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:49,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:01:50,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:51,152 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:01:51,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:01:52,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:54,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:55,260 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:01:56,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:01:56,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:01:58,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:00,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:00,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:02:01,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:02:02,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:04,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:05,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:02:06,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:06,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:02:07,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:02:07,776 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:02:07,777 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:02:07,778 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:02:08,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:08,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:02:09,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:10,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:11,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:02:11,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:02:12,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:14,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:16,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:16,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:02:16,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:02:18,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:20,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:21,166 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:02:21,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:02:21,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:02:22,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:24,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:26,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:26,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:02:27,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:02:28,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:30,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:31,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:02:32,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:32,658 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:02:34,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:36,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:36,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:02:37,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:02:38,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:40,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:41,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:02:42,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:43,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:02:44,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:46,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:46,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:02:48,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:48,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:02:50,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:51,156 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:02:51,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:02:52,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:52,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:02:52,272 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:02:52,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:02:52,275 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:02:53,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:02:54,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:02:54,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:55,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:56,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:56,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:02:58,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:02:59,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:03:00,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:01,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:03:02,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:04,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:04,740 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:03:06,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:06,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:03:08,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:10,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:03:10,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:11,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:03:12,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:14,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:15,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:03:16,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:16,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:03:18,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:20,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:20,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:03:21,159 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:03:21,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:03:22,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:24,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:26,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:26,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:03:26,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:03:28,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:30,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:31,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:03:31,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:03:32,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:34,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:36,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:36,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:03:36,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:03:37,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:03:37,021 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:03:37,021 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:03:37,023 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:03:37,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:03:38,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:39,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:40,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:41,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:03:42,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:42,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:03:44,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:46,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:46,711 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:03:47,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:03:48,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:50,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:51,161 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:03:51,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:03:52,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:53,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:03:54,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:56,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:03:56,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:58,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:03:58,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:04:00,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:01,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:04:02,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:03,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:04:04,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:06,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:06,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:04:08,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:09,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:04:10,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:11,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:04:12,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:14,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:14,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:04:16,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:16,715 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:04:18,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:19,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:04:20,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:21,174 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:04:21,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:04:21,691 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:04:21,692 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:04:21,693 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:04:21,727 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:04:22,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:22,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:04:23,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:24,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:25,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:04:26,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:04:26,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:28,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:30,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:30,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:04:31,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:04:32,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:34,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:35,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:04:36,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:36,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:04:38,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:40,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:41,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:04:41,320 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:04:42,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:44,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:46,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:46,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:04:46,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:04:48,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:50,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:51,166 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:04:51,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:04:52,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:04:52,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:54,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:56,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:04:56,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:04:57,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:04:58,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:00,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:01,760 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:05:02,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:02,739 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:05:04,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:06,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:05:06,349 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:05:06,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:05:06,351 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:05:06,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:06,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:05:06,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:05:08,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:05:08,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:09,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:10,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:11,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:05:12,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:13,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:05:14,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:16,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:16,762 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:05:18,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:18,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:05:20,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:21,168 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:05:21,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:05:22,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:24,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:05:24,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:26,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:05:26,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:28,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:29,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:05:30,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:31,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:05:32,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:34,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:34,906 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:05:36,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:36,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:05:38,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:40,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:05:40,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:41,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:05:42,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:44,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:45,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:05:46,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:46,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:05:48,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:50,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:50,938 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:05:50,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:05:50,971 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:05:50,972 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:05:50,975 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:05:51,170 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:05:51,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:05:51,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:05:52,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:54,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:56,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:05:56,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:05:56,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:05:58,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:00,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:01,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:06:01,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:06:02,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:04,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:06,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:06,788 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:06:06,984 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:06:08,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:10,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:11,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:06:12,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:06:12,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:14,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:16,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:16,789 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:06:17,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:06:18,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:20,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:21,184 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:06:21,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:06:22,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:23,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:06:24,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:26,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:06:26,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:28,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:06:28,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:30,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:31,794 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:06:32,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:33,746 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:06:34,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:35,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:06:35,564 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:06:35,565 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:06:35,566 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:06:36,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:36,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:06:36,805 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:06:38,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:39,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:06:40,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:41,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:06:42,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:44,469 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:06:44,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:46,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:46,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:06:48,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:49,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:06:50,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:51,174 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:06:51,807 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:06:52,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:54,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:55,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:06:56,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:06:56,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:06:58,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:00,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:00,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:07:01,821 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:07:02,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:04,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:05,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:07:06,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:06,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:07:08,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:10,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:11,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:07:11,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:07:12,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:14,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:16,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:16,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:07:16,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:07:18,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:20,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:07:20,220 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:07:20,220 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:07:20,222 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:07:20,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:20,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:07:21,177 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:07:21,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:07:21,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:07:22,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:24,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:26,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:07:26,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:27,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:07:28,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:30,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:31,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:07:32,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:32,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:07:34,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:36,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:36,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:07:38,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:07:38,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:40,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:41,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:07:42,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:43,459 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:07:44,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:46,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:46,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:07:48,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:48,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:07:50,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:51,179 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:07:51,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:07:52,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:54,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:07:54,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:56,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:07:56,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:58,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:07:59,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:08:00,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:01,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:08:02,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:04,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:04,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:08:04,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:08:04,872 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:08:04,873 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:08:04,874 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:08:05,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:08:06,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:06,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:08:08,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:10,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:08:10,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:11,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:08:12,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:14,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:15,567 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:08:16,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:16,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:08:18,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:20,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:20,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:08:21,181 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:08:21,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:08:22,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:24,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:26,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:08:26,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:08:26,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:28,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:30,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:31,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:08:31,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:08:32,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:34,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:36,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:36,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:08:36,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:08:38,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:40,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:41,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:08:42,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:08:42,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:44,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:46,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:46,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:08:47,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:08:48,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:49,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:08:49,491 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:08:49,491 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:08:49,493 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:08:49,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:08:50,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:51,183 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:08:51,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:08:52,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:53,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:08:54,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:56,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:08:56,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:08:58,407 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:08:58,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:00,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:01,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:09:02,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:03,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:09:04,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:06,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:06,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:09:08,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:09,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:09:10,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:11,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:09:12,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:14,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:09:14,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:16,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:16,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:09:18,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:19,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:09:20,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:21,186 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:09:21,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:09:22,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:24,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:25,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:09:26,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:09:26,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:28,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:30,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:09:30,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:31,889 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:09:32,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:33,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:34,125 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:09:34,127 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:09:34,127 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:09:34,130 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:09:34,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:09:35,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:09:35,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:36,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:09:36,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:37,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:39,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:41,275 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:09:41,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:09:41,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:43,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:45,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:46,627 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:09:46,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:09:47,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:49,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:51,188 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:09:51,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:09:51,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:51,987 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:09:53,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:55,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:56,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:09:57,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:09:57,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:09:59,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:01,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:10:01,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:02,669 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:10:03,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:05,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:06,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:10:07,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:08,045 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:10:09,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:11,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:10:11,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:13,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:10:13,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:15,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:16,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:10:18,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:18,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:10:18,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:10:18,804 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:10:18,804 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:10:18,806 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:10:19,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:10:20,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:21,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:21,190 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:10:21,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:10:22,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:24,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:24,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:10:26,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:26,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:10:28,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:29,461 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:10:30,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:31,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:10:32,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:34,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:34,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:10:36,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:36,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:10:38,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:40,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:40,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:10:41,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:10:42,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:44,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:45,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:10:46,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:46,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:10:48,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:50,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:50,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:10:51,192 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:10:51,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:10:52,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:54,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:56,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:10:56,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:10:56,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:10:58,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:00,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:01,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:11:01,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:11:02,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:03,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:11:03,379 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:11:03,380 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:11:03,381 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:11:04,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:04,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:11:05,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:06,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:06,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:11:06,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:11:08,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:10,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:11,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:11:12,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:12,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:11:14,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:16,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:16,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:11:17,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:11:18,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:20,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:21,194 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:11:21,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:11:22,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:23,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:11:24,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:26,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:26,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:11:28,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:28,361 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:11:30,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:31,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:11:32,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:33,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:11:34,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:36,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:36,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:11:38,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:39,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:11:40,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:41,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:11:42,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:44,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:44,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:11:46,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:46,987 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:11:47,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:11:47,961 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:11:47,961 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:11:47,963 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:11:48,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:48,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:11:49,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:49,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:11:50,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:51,197 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:11:52,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:11:52,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:54,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:55,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:11:56,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:11:56,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:11:58,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:00,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:00,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:12:02,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:12:02,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:04,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:05,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:12:06,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:07,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:12:08,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:10,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:11,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:12:11,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:12:12,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:14,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:16,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:16,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:12:17,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:12:18,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:20,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:21,199 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:12:21,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:12:22,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:12:22,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:24,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:26,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:26,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:12:27,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:12:28,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:30,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:32,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:12:32,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:32,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:12:32,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:12:32,653 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:12:32,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:12:32,656 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:12:33,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:12:34,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:35,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:36,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:37,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:12:37,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:12:38,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:40,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:41,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:12:42,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:43,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:12:44,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:46,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:47,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:12:48,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:48,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:12:50,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:51,201 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:12:52,030 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:12:52,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:53,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:12:54,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:56,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:56,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:12:58,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:12:59,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:13:00,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:02,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:13:02,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:04,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:04,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:13:06,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:07,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:13:08,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:10,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:13:10,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:11,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:13:12,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:14,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:15,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:13:16,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:17,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:13:17,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:13:17,330 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:13:17,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:13:17,332 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:13:18,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:18,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:13:19,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:20,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:20,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:13:21,204 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:13:22,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:13:22,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:24,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:26,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:26,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:13:26,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:13:28,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:30,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:31,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:13:32,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:13:32,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:34,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:36,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:36,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:13:37,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:13:38,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:40,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:41,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:13:42,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:13:42,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:44,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:46,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:47,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:13:47,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:13:48,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:50,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:51,205 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:13:52,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:13:52,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:52,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:13:54,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:56,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:13:56,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:13:58,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:13:58,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:00,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:01,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:14:01,829 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:14:01,829 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:14:01,831 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:14:02,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:14:02,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:02,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:14:03,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:03,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:14:04,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:06,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:07,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:14:08,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:08,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:14:10,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:11,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:14:12,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:14,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:14,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:14:16,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:17,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:14:18,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:19,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:14:20,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:21,218 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:14:22,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:14:22,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:24,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:24,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:14:26,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:26,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:14:28,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:30,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:30,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:14:32,085 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:14:32,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:34,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:35,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:14:36,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:37,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:14:38,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:40,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:41,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:14:41,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:14:42,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:44,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:46,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:46,427 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:14:46,459 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:14:46,460 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:14:46,461 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:14:46,462 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:14:47,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:14:47,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:14:48,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:49,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:50,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:51,209 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:14:51,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:14:52,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:14:52,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:54,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:56,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:14:56,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:14:57,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:14:58,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:00,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:02,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:15:02,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:02,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:15:04,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:06,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:07,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:15:07,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:15:08,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:10,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:11,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:15:12,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:13,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:15:14,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:16,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:17,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:15:18,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:18,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:15:20,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:21,211 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:15:22,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:15:22,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:23,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:15:24,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:26,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:15:26,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:28,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:29,260 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:15:30,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:31,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:15:31,083 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:15:31,083 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:15:31,085 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:15:31,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:15:32,117 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:15:32,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:34,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:34,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:15:36,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:37,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:15:38,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:40,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:15:40,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:41,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:15:42,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:44,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:45,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:15:46,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:47,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:15:48,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:50,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:50,736 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:15:51,214 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:15:52,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:15:52,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:54,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:56,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:15:56,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:15:56,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:15:58,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:00,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:01,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:16:02,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:16:02,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:04,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:06,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:06,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:16:07,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:16:08,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:10,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:11,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:16:12,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:16:12,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:14,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:15,727 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:16:15,729 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:16:15,730 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:16:15,731 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:16:16,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:16,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:16:17,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:16:17,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:16:18,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:20,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:21,216 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:16:22,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:16:22,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:22,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:16:24,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:26,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:16:26,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:28,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:16:28,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:30,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:32,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:16:32,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:33,642 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:16:34,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:36,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:37,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:16:38,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:38,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:16:40,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:41,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:16:42,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:44,356 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:16:44,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:46,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:47,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:16:48,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:49,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:16:50,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:51,219 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:16:52,156 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:16:52,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:54,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:55,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:16:56,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:16:56,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:16:58,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:00,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:17:00,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:17:00,414 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:17:00,414 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:17:00,416 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:17:00,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:00,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:17:02,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:17:02,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:04,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:05,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:17:06,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:07,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:17:08,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:10,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:11,085 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:17:11,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:17:12,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:14,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:16,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:17:16,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:17,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:17:18,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:20,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:21,220 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:17:21,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:17:22,168 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:17:22,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:24,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:26,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:17:26,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:27,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:17:28,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:30,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:32,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:17:32,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:32,522 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:17:34,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:36,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:37,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:17:37,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:17:38,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:40,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:41,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:17:42,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:43,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:17:44,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:45,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:17:45,018 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:17:45,019 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:17:45,021 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:17:45,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:17:46,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:47,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:17:48,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:48,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:17:50,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:51,223 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:17:52,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:17:52,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:53,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:17:54,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:56,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:17:56,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:58,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:17:59,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:18:00,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:02,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:18:02,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:04,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:04,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:18:06,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:07,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:18:08,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:09,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:18:10,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:11,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:18:12,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:14,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:15,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:18:16,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:17,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:18:18,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:20,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:20,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:18:21,225 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:18:22,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:18:22,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:24,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:26,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:18:26,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:18:26,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:28,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:29,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:18:29,647 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:18:29,648 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:18:29,650 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:18:30,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:30,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:18:31,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:18:32,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:18:32,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:34,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:36,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:36,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:18:37,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:18:38,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:40,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:41,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:18:42,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:18:42,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:44,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:46,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:47,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:18:47,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:18:48,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:50,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:51,227 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:18:52,233 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:18:52,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:52,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:18:54,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:56,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:18:56,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:18:58,202 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:18:58,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:00,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:02,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:19:02,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:03,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:19:04,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:06,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:07,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:19:08,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:08,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:19:10,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:11,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:19:12,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:14,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:19:14,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:19:14,260 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:19:14,260 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:19:14,263 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:19:14,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:14,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:19:16,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:19:16,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:18,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:19,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:19:20,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:21,229 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:19:21,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:19:22,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:24,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:24,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:19:26,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:19:26,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:28,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:30,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:19:30,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:31,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:19:32,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:34,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:35,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:19:36,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:19:36,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:38,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:40,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:41,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:19:41,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:19:42,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:44,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:46,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:19:46,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:19:46,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:48,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:50,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:51,231 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:19:51,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:19:51,699 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:19:52,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:54,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:56,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:19:56,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:57,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:19:58,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:19:58,857 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:19:58,857 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:19:58,860 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:19:58,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:19:58,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:20:00,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:01,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:20:02,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:20:02,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:04,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:06,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:20:06,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:07,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:20:08,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:10,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:11,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:20:12,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:13,166 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:20:14,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:16,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:20:16,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:18,536 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:20:18,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:20,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:21,233 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:20:21,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:20:22,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:23,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:20:24,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:26,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:20:26,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:28,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:29,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:20:30,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:31,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:20:32,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:34,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:20:34,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:36,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:20:36,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:38,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:39,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:20:40,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:41,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:20:42,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:43,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:20:43,543 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:20:43,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:20:43,546 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:20:43,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:20:44,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:45,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:20:46,356 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:20:46,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:48,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:50,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:20:50,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:51,236 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:20:51,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:20:52,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:54,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:56,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:20:56,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:20:56,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:20:58,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:00,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:01,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:21:01,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:21:02,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:04,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:06,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:21:06,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:21:06,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:08,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:10,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:11,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:21:12,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:21:12,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:14,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:16,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:21:16,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:17,449 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:21:18,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:20,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:21,238 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:21:21,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:21:22,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:21:22,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:24,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:26,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:21:26,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:28,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:21:28,166 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:21:28,168 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:21:28,168 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:21:28,170 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:21:29,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:29,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:21:31,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:31,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:21:33,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:33,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:21:35,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:36,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:21:37,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:38,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:21:39,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:41,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:41,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:21:43,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:44,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:21:45,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:46,422 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:21:47,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:49,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:49,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:21:51,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:51,240 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:21:51,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:21:53,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:54,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:21:55,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:56,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:21:57,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:21:59,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:00,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:22:01,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:01,459 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:22:03,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:05,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:05,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:22:06,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:22:07,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:09,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:10,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:22:11,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:11,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:22:12,810 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:22:12,824 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:22:12,824 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:22:12,826 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:22:13,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:13,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:22:15,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:16,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:22:16,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:22:17,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:19,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:21,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:21,242 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:22:21,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:22:21,674 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:22:23,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:25,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:26,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:22:27,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:22:27,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:29,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:31,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:31,519 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:22:32,432 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:22:33,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:35,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:36,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:22:37,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:37,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:22:39,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:41,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:41,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:22:43,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:43,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:22:45,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:46,521 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:22:47,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:48,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:22:49,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:51,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:51,244 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:22:51,522 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:22:53,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:53,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:22:55,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:56,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:22:57,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:57,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:22:57,439 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:22:57,439 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:22:57,441 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:22:58,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:22:59,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:22:59,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:23:01,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:01,523 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:23:03,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:04,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:23:05,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:06,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:23:07,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:09,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:09,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:23:11,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:11,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:23:13,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:15,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:15,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:23:16,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:23:17,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:19,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:20,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:23:21,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:21,246 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:23:21,526 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:23:23,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:25,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:25,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:23:26,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:23:27,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:29,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:31,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:31,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:23:31,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:23:33,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:35,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:36,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:23:36,736 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:23:37,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:39,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:41,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:41,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:23:42,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:23:42,138 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:23:42,139 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:23:42,139 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:23:42,141 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:23:43,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:43,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:23:45,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:46,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:23:47,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:47,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:23:49,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:51,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:51,248 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:23:51,530 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:23:52,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:23:53,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:55,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:56,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:23:57,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:23:58,107 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:23:59,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:01,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:01,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:24:03,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:03,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:24:05,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:06,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:24:07,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:08,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:24:09,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:11,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:11,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:24:12,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:14,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:24:14,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:16,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:16,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:24:18,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:19,466 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:24:20,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:21,251 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:24:21,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:24:22,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:24,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:24,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:24:26,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:26,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:24:26,658 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:24:26,659 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:24:26,659 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:24:26,661 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:24:27,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:24:28,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:30,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:30,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:24:31,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:24:32,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:34,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:35,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:24:36,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:36,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:24:38,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:40,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:40,949 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:24:41,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:24:42,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:44,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:46,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:46,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:24:46,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:24:48,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:50,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:51,253 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:24:51,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:24:51,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:24:52,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:54,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:56,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:24:56,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:24:57,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:24:58,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:00,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:01,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:25:02,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:02,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:25:04,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:06,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:06,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:25:07,663 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:25:08,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:10,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:11,255 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:25:11,257 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:25:11,257 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:25:11,259 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:25:11,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:25:12,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:12,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:25:13,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:25:14,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:16,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:16,642 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:25:18,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:18,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:25:20,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:21,255 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:25:21,649 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:25:22,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:23,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:25:24,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:26,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:26,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:25:28,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:29,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:25:30,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:31,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:25:32,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:34,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:34,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:25:36,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:36,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:25:38,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:39,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:25:40,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:41,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:25:42,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:44,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:45,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:25:46,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:46,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:25:48,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:50,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:50,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:25:51,256 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:25:51,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:25:52,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:54,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:55,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:25:55,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:25:55,991 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:25:55,991 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:25:55,994 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:25:56,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:25:56,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:25:56,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:25:58,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:00,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:01,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:26:01,675 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:26:02,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:04,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:06,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:06,622 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:26:06,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:26:08,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:10,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:11,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:26:11,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:26:12,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:14,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:16,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:16,688 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:26:17,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:26:18,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:20,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:21,259 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:26:21,688 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:26:22,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:22,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:26:24,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:26,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:26:26,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:27,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:26:28,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:30,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:31,694 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:26:32,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:33,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:26:34,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:36,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:36,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:26:38,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:38,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:26:40,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:40,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:26:40,486 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:26:40,486 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:26:40,488 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:26:41,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:26:41,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:26:42,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:44,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:26:44,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:46,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:46,726 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:26:48,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:49,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:26:50,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:51,261 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:26:51,726 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:26:52,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:54,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:54,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:26:56,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:26:56,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:26:58,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:00,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:27:00,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:01,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:27:02,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:04,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:05,407 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:27:06,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:06,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:27:08,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:10,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:10,740 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:27:11,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:27:12,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:14,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:16,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:27:16,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:16,729 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:27:18,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:20,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:21,263 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:27:21,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:27:21,730 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:27:22,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:24,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:25,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:27:25,015 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:27:25,015 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:27:25,017 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:27:25,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:27:26,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:27:26,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:26,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:27:28,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:30,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:31,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:27:32,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:27:32,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:34,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:36,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:36,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:27:37,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:27:38,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:40,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:41,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:27:42,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:42,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:27:44,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:46,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:46,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:27:48,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:27:48,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:50,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:51,265 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:27:51,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:27:52,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:53,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:27:54,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:56,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:27:56,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:58,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:27:58,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:28:00,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:01,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:28:02,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:04,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:28:04,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:06,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:06,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:28:08,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:09,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:28:09,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:28:09,672 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:28:09,672 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:28:09,674 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:28:10,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:10,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:28:11,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:28:12,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:14,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:14,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:28:16,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:16,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:28:18,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:20,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:28:20,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:21,268 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:28:21,748 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:28:22,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:24,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:25,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:28:26,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:28:26,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:28,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:30,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:30,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:28:31,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:28:32,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:34,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:36,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:28:36,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:36,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:28:38,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:40,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:41,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:28:41,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:28:42,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:44,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:46,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:46,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:28:46,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:28:48,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:50,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:51,270 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:28:51,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:28:52,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:28:52,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:54,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:28:54,203 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:28:54,203 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:28:54,205 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:28:54,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:54,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:28:56,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:28:56,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:28:57,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:28:58,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:00,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:01,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:29:02,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:03,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:29:04,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:06,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:06,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:29:08,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:29:08,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:10,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:11,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:29:12,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:13,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:29:14,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:16,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:16,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:29:18,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:19,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:29:20,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:21,272 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:29:21,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:29:22,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:24,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:29:24,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:26,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:29:26,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:28,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:29,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:29:30,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:31,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:29:32,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:34,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:35,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:29:36,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:36,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:29:38,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:38,795 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:29:38,796 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:29:38,797 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:29:38,798 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:29:39,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:29:40,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:29:40,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:41,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:29:42,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:44,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:45,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:29:46,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:29:46,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:48,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:50,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:51,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:29:51,274 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:29:51,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:29:52,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:54,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:56,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:29:56,558 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:29:56,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:29:58,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:00,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:01,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:30:01,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:30:02,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:04,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:06,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:30:06,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:07,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:30:08,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:10,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:11,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:30:12,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:30:12,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:14,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:16,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:30:16,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:17,991 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:30:18,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:20,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:21,276 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:30:21,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:30:22,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:23,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:30:23,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:30:23,403 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:30:23,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:30:23,406 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:30:23,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:30:24,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:26,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:30:26,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:28,738 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:30:28,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:30,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:31,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:30:32,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:34,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:30:34,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:36,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:30:36,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:38,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:39,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:30:40,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:41,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:30:42,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:44,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:30:44,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:46,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:30:46,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:48,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:50,109 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:30:50,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:51,279 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:30:51,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:30:52,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:54,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:55,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:30:56,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:30:56,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:30:58,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:00,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:31:00,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:01,838 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:31:02,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:04,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:06,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:31:06,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:31:06,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:07,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:31:07,936 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:31:07,936 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:31:07,938 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:31:08,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:08,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:31:10,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:11,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:31:11,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:31:12,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:14,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:16,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:31:16,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:31:16,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:18,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:20,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:21,281 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:31:21,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:31:22,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:31:22,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:24,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:26,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:31:26,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:27,631 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:31:28,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:30,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:31,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:31:32,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:32,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:31:34,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:36,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:31:36,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:38,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:31:38,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:40,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:41,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:31:42,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:43,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:31:44,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:46,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:31:46,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:48,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:49,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:31:50,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:51,283 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:31:51,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:31:52,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:31:52,610 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:31:52,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:31:52,612 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:31:52,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:52,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:31:54,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:31:54,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:56,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:31:56,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:58,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:31:59,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:32:00,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:01,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:32:02,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:04,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:05,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:32:06,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:32:06,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:08,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:10,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:32:10,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:11,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:32:12,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:14,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:15,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:32:16,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:32:16,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:18,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:20,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:21,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:32:21,285 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:32:21,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:32:22,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:24,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:26,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:32:26,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:32:26,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:28,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:31,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:31,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:32:31,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:32:33,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:35,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:36,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:32:37,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:37,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:32:37,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:32:37,260 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:32:37,261 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:32:37,262 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:32:38,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:32:39,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:41,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:41,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:32:42,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:32:43,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:45,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:46,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:32:47,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:47,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:32:49,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:51,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:51,288 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:32:51,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:32:53,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:53,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:32:55,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:56,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:32:57,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:32:58,582 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:32:59,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:01,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:01,906 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:33:03,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:03,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:33:05,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:06,906 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:33:07,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:09,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:09,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:33:11,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:11,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:33:13,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:14,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:33:15,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:16,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:33:17,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:19,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:20,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:33:21,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:21,290 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:33:21,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:33:21,880 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:33:21,880 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:33:21,882 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:33:21,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:33:22,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:33:23,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:25,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:25,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:33:26,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:33:27,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:29,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:30,770 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:33:31,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:31,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:33:33,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:35,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:36,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:33:36,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:33:37,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:39,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:41,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:41,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:33:41,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:33:43,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:45,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:46,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:33:46,950 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:33:47,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:49,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:51,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:51,292 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:33:51,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:33:52,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:33:53,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:55,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:56,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:33:57,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:33:57,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:33:59,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:01,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:01,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:34:02,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:34:03,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:05,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:06,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:34:06,562 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:34:06,562 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:34:06,564 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:34:06,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:34:07,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:07,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:34:08,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:34:09,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:11,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:11,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:34:13,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:13,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:34:15,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:16,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:34:17,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:19,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:34:19,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:21,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:21,294 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:34:21,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:34:23,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:24,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:34:25,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:26,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:34:27,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:29,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:29,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:34:31,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:31,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:34:33,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:35,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:34:35,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:36,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:34:37,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:39,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:40,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:34:41,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:41,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:34:43,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:45,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:45,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:34:46,973 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:34:47,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:49,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:51,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:34:51,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:34:51,067 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:34:51,067 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:34:51,069 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:34:51,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:51,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:34:51,339 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:34:51,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:34:53,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:55,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:56,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:34:56,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:34:57,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:34:59,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:01,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:01,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:35:01,975 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:35:03,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:05,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:06,976 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:35:07,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:35:07,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:09,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:11,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:11,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:35:12,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:35:13,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:15,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:17,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:35:17,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:17,865 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:35:19,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:21,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:21,341 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:35:22,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:35:23,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:23,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:35:25,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:26,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:35:27,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:28,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:35:29,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:31,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:32,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:35:33,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:33,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:35:35,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:35,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:35:35,696 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:35:35,697 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:35:35,699 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:35:36,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:35:37,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:35:37,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:39,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:39,235 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:35:41,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:41,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:35:43,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:44,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:35:45,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:47,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:35:47,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:49,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:49,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:35:51,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:51,343 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:35:52,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:35:53,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:55,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:55,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:35:56,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:35:57,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:35:59,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:00,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:36:01,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:02,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:36:03,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:05,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:06,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:36:07,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:36:07,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:09,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:11,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:11,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:36:11,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:36:13,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:15,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:16,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:36:17,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:36:17,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:19,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:20,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:36:20,309 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:36:20,310 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:36:20,311 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:36:21,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:21,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:36:21,345 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:36:22,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:36:22,065 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:36:23,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:25,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:26,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:36:27,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:27,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:36:29,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:31,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:32,030 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:36:32,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:36:33,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:35,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:37,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:36:37,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:38,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:36:39,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:41,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:41,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:36:43,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:43,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:36:45,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:47,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:36:47,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:48,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:36:49,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:51,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:51,348 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:36:52,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:36:53,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:54,192 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:36:55,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:56,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:36:57,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:59,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:36:59,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:37:01,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:02,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:37:03,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:04,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:37:04,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:37:04,915 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:37:04,915 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:37:04,916 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:37:05,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:05,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:37:07,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:37:07,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:09,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:10,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:37:11,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:11,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:37:13,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:15,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:15,627 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:37:17,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:37:17,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:19,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:21,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:37:21,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:21,350 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:37:22,043 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:37:23,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:25,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:26,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:37:26,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:37:27,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:29,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:31,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:31,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:37:32,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:37:33,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:35,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:37,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:37:37,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:37:37,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:39,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:41,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:37:41,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:42,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:37:43,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:45,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:47,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:37:47,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:47,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:37:49,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:49,598 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:37:49,600 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:37:49,600 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:37:49,602 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:37:50,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:37:51,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:51,363 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:37:52,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:37:53,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:37:53,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:55,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:56,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:37:57,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:37:58,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:37:59,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:01,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:02,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:38:03,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:03,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:38:05,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:07,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:38:07,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:09,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:38:09,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:11,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:38:11,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:13,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:14,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:38:15,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:17,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:38:17,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:19,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:19,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:38:21,354 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:38:21,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:22,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:38:23,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:25,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:38:25,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:26,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:38:27,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:29,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:30,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:38:31,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:32,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:38:33,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:34,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:38:34,285 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:38:34,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:38:34,288 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:38:34,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:38:35,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:36,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:38:37,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:38:37,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:39,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:41,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:38:41,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:38:41,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:43,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:45,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:46,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:38:47,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:38:47,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:48,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:50,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:51,357 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:38:52,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:38:52,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:38:52,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:54,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:56,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:38:56,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:38:57,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:38:58,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:00,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:02,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:39:02,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:02,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:39:04,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:06,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:07,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:39:08,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:39:08,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:10,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:11,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:39:12,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:13,469 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:39:14,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:16,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:17,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:39:18,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:18,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:39:18,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:39:18,885 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:39:18,885 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:39:18,887 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:39:19,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:39:20,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:21,359 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:39:22,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:39:22,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:24,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:39:24,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:26,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:39:26,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:28,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:29,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:39:30,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:32,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:39:32,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:34,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:34,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:39:36,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:37,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:39:38,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:40,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:39:40,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:41,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:39:42,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:44,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:45,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:39:46,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:47,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:39:48,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:50,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:50,950 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:39:51,361 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:39:52,141 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:39:52,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:54,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:56,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:39:56,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:39:56,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:39:58,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:00,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:01,631 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:40:02,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:40:02,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:03,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:40:03,443 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:40:03,444 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:40:03,445 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:40:03,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:40:04,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:06,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:06,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:40:07,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:40:08,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:10,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:11,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:40:12,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:40:12,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:14,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:16,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:17,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:40:17,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:40:18,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:20,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:21,364 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:40:22,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:40:22,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:23,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:40:24,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:26,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:40:26,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:28,632 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:40:28,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:30,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:32,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:40:32,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:33,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:40:34,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:36,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:37,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:40:38,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:39,369 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:40:40,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:41,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:40:42,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:44,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:40:44,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:46,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:47,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:40:48,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:40:48,305 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:40:48,306 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:40:48,308 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:40:48,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:48,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:40:50,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:40:50,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:51,366 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:40:52,156 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:40:52,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:54,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:55,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:40:56,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:40:56,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:40:58,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:00,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:41:00,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:02,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:41:02,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:04,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:06,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:41:06,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:07,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:41:08,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:10,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:11,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:41:11,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:41:12,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:14,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:16,851 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:41:16,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:17,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:41:18,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:20,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:21,368 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:41:22,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:41:22,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:41:22,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:24,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:26,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:41:26,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:27,573 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:41:28,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:30,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:32,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:41:32,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:32,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:41:32,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:41:32,946 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:41:32,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:41:32,947 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:41:33,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:41:34,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:36,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:37,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:41:38,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:41:38,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:40,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:41,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:41:42,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:43,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:41:44,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:46,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:47,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:41:48,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:49,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:41:50,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:51,370 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:41:52,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:41:52,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:54,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:41:54,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:56,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:41:56,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:58,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:41:59,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:42:00,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:02,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:42:02,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:04,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:05,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:42:06,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:07,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:42:08,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:10,469 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:42:10,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:11,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:42:12,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:14,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:15,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:42:16,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:17,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:42:17,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:42:17,632 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:42:17,632 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:42:17,634 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:42:17,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:42:18,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:20,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:21,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:42:21,372 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:42:22,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:42:22,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:24,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:26,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:42:26,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:42:26,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:28,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:30,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:31,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:42:32,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:42:32,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:34,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:36,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:37,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:42:37,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:42:38,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:40,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:41,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:42:42,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:42:42,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:44,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:46,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:47,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:42:47,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:42:48,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:50,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:51,375 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:42:52,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:42:52,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:53,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:42:54,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:56,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:42:56,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:42:58,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:42:58,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:00,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:02,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:43:02,183 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:43:02,184 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:43:02,185 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:43:02,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:43:02,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:02,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:43:03,938 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:43:04,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:06,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:07,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:43:08,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:09,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:43:10,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:11,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:43:12,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:14,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:43:14,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:16,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:17,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:43:18,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:19,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:43:20,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:21,377 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:43:22,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:43:22,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:24,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:25,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:43:26,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:43:26,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:28,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:30,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:43:30,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:32,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:43:32,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:35,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:35,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:43:37,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:37,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:43:39,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:41,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:41,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:43:41,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:43:43,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:45,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:46,637 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:43:46,669 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:43:46,671 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:43:46,671 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:43:46,673 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:43:47,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:47,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:43:47,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:43:49,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:51,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:51,379 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:43:51,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:43:52,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:43:53,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:55,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:56,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:43:57,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:43:57,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:43:59,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:01,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:02,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:44:02,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:44:03,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:05,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:07,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:07,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:44:07,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:44:09,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:11,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:11,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:44:13,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:13,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:44:15,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:17,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:17,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:44:18,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:44:19,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:21,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:21,381 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:44:22,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:44:23,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:24,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:44:25,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:26,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:44:27,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:29,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:29,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:44:31,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:31,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:44:31,251 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:44:31,252 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:44:31,253 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:44:32,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:44:32,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:44:33,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:34,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:44:35,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:37,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:37,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:44:39,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:40,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:44:41,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:41,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:44:43,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:45,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:45,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:44:47,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:47,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:44:49,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:50,823 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:44:51,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:51,384 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:44:52,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:44:53,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:55,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:56,202 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:44:56,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:44:57,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:44:59,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:01,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:01,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:45:01,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:45:03,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:05,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:06,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:45:06,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:45:07,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:09,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:11,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:11,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:45:12,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:45:13,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:15,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:15,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:45:15,885 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:45:15,885 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:45:15,888 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:45:16,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:45:16,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:45:17,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:17,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:45:19,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:21,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:21,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:45:21,385 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:45:22,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:45:23,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:25,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:26,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:45:27,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:28,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:45:29,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:31,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:31,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:45:33,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:33,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:45:35,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:36,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:45:37,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:39,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:45:39,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:41,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:41,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:45:43,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:44,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:45:45,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:46,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:45:47,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:49,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:49,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:45:51,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:51,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:45:51,387 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:45:53,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:55,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:45:55,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:56,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:45:57,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:45:59,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:00,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:46:00,499 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:46:00,501 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:46:00,501 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:46:00,504 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:46:01,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:01,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:46:01,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:46:03,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:05,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:05,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:46:06,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:46:07,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:09,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:11,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:46:11,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:11,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:46:13,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:15,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:16,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:46:16,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:46:17,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:19,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:21,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:21,390 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:46:21,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:46:21,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:46:23,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:25,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:26,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:46:27,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:27,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:46:29,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:31,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:31,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:46:32,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:46:33,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:35,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:36,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:46:37,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:37,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:46:39,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:41,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:41,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:46:43,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:43,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:46:45,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:46:45,139 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:46:45,140 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:46:45,141 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:46:45,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:45,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:46:46,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:46:47,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:48,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:46:49,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:51,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:51,392 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:46:51,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:46:53,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:54,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:46:55,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:56,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:46:57,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:59,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:46:59,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:47:01,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:01,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:47:03,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:04,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:47:05,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:06,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:47:07,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:09,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:10,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:47:11,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:11,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:47:13,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:15,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:15,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:47:16,432 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:47:17,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:19,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:20,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:47:21,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:21,395 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:47:21,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:47:23,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:25,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:26,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:47:26,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:47:27,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:29,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:29,734 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:47:29,736 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:47:29,736 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:47:29,739 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:47:30,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:47:31,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:31,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:47:31,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:47:33,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:35,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:36,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:47:36,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:47:37,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:39,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:41,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:41,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:47:42,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:47:43,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:45,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:46,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:47:47,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:47,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:47:49,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:51,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:51,397 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:47:51,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:47:52,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:47:53,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:55,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:56,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:47:57,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:47:58,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:47:59,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:01,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:01,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:48:03,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:03,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:48:05,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:06,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:48:07,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:09,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:48:09,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:11,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:11,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:48:13,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:14,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:48:14,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:48:14,397 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:48:14,397 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:48:14,399 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:48:15,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:15,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:48:16,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:48:17,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:19,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:19,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:48:21,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:21,399 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:48:21,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:48:23,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:25,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:48:25,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:26,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:48:27,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:29,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:30,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:48:31,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:31,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:48:33,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:35,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:35,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:48:36,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:48:37,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:39,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:41,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:48:41,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:48:41,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:43,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:45,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:46,466 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:48:46,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:48:47,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:49,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:51,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:51,402 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:48:51,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:48:51,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:48:53,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:55,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:56,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:48:57,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:48:57,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:59,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:48:59,003 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:48:59,003 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:48:59,005 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:48:59,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:48:59,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:49:01,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:01,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:49:02,558 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:49:03,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:05,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:06,476 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:49:07,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:07,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:49:09,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:11,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:49:11,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:13,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:49:13,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:15,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:16,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:49:17,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:18,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:49:19,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:21,404 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:49:21,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:21,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:49:23,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:23,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:49:25,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:26,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:49:27,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:29,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:49:29,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:31,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:49:31,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:33,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:34,715 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:49:35,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:36,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:49:37,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:39,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:40,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:49:41,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:49:41,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:43,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:43,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:49:43,655 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:49:43,656 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:49:43,657 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:49:44,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:49:45,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:49:45,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:46,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:49:47,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:49,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:50,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:49:51,416 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:49:51,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:49:51,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:53,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:55,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:56,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:49:56,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:49:57,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:49:59,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:01,429 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:50:01,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:50:01,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:03,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:05,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:06,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:50:06,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:50:07,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:09,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:11,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:50:11,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:12,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:50:13,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:14,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:16,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:50:16,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:17,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:50:18,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:20,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:21,408 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:50:21,511 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:50:22,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:22,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:50:24,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:26,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:50:26,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:28,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:50:28,263 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:50:28,265 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:50:28,265 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:50:28,267 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:50:28,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:28,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:50:30,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:31,519 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:50:32,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:33,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:50:34,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:36,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:50:36,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:38,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:38,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:50:40,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:41,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:50:42,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:44,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:50:44,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:46,530 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:50:46,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:48,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:49,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:50:50,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:51,410 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:50:51,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:50:52,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:54,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:54,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:50:56,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:50:56,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:50:58,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:00,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:51:00,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:01,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:51:02,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:04,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:05,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:51:06,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:51:06,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:08,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:10,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:11,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:51:11,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:51:12,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:12,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:51:12,863 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:51:12,863 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:51:12,866 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:51:13,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:51:14,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:16,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:51:16,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:51:16,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:18,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:20,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:21,413 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:51:21,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:51:21,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:51:22,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:24,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:26,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:51:26,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:27,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:51:28,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:30,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:31,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:51:32,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:51:32,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:34,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:36,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:51:36,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:37,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:51:38,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:40,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:41,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:51:42,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:43,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:51:44,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:46,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:51:46,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:48,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:51:48,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:50,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:51,414 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:51:51,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:51:52,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:53,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:51:54,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:56,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:51:56,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:57,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:51:57,399 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:51:57,399 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:51:57,401 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:51:57,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:51:58,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:51:59,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:52:00,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:01,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:52:02,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:04,469 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:52:04,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:06,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:52:06,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:08,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:09,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:52:10,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:11,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:52:12,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:14,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:15,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:52:16,551 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:52:16,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:18,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:20,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:52:20,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:21,416 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:52:21,559 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:52:22,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:24,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:25,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:52:26,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:52:26,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:28,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:30,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:31,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:52:31,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:52:32,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:34,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:36,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:52:36,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:52:36,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:38,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:40,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:41,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:52:41,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:52:41,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:52:41,891 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:52:41,891 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:52:41,893 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:52:42,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:42,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:52:44,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:46,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:52:46,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:47,192 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:52:48,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:50,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:51,419 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:52:51,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:52:52,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:52:52,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:54,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:56,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:52:56,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:52:57,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:52:58,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:00,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:01,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:53:02,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:03,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:53:04,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:06,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:53:06,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:08,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:53:08,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:10,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:11,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:53:12,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:14,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:53:14,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:16,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:53:16,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:18,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:19,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:53:20,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:21,421 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:53:21,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:53:22,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:24,725 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:53:24,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:26,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:53:26,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:53:26,551 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:53:26,551 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:53:26,553 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:53:26,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:26,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:53:28,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:30,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:53:30,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:31,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:53:32,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:34,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:35,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:53:36,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:53:36,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:38,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:40,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:53:40,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:41,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:53:42,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:44,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:46,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:53:46,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:53:46,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:48,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:50,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:51,423 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:53:51,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:53:51,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:53:52,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:54,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:56,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:53:56,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:53:56,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:53:58,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:00,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:01,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:54:02,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:54:02,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:04,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:06,629 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:54:06,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:07,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:54:08,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:10,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:11,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:54:11,069 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:54:11,069 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:54:11,071 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:54:11,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:54:11,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:54:12,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:54:12,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:14,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:16,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:54:16,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:18,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:54:18,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:20,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:21,425 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:54:21,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:54:22,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:23,512 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:54:24,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:26,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:54:26,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:28,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:54:28,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:30,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:31,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:54:32,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:34,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:54:34,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:36,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:54:36,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:38,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:39,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:54:40,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:41,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:54:42,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:44,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:54:44,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:46,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:54:46,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:48,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:50,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:54:50,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:51,428 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:54:51,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:54:52,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:54,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:55,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:54:55,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:54:55,520 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:54:55,521 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:54:55,522 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:54:55,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:54:56,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:54:56,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:54:58,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:00,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:55:00,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:01,682 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:55:02,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:04,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:06,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:55:06,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:55:06,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:08,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:10,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:11,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:55:11,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:55:12,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:14,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:16,701 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:55:16,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:55:16,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:18,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:20,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:21,430 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:55:21,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:55:22,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:55:22,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:24,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:26,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:55:26,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:27,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:55:28,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:30,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:31,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:55:32,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:32,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:55:34,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:36,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:55:36,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:38,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:55:38,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:40,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:55:40,147 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:55:40,147 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:55:40,149 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:55:40,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:40,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:55:41,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:55:42,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:43,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:55:45,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:46,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:55:47,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:49,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:49,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:55:51,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:51,432 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:55:51,734 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:55:53,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:54,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:55:55,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:56,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:55:57,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:59,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:55:59,740 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:56:01,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:01,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:56:03,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:05,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:05,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:56:06,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:56:07,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:09,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:10,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:56:11,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:11,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:56:13,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:15,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:15,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:56:16,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:56:17,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:19,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:21,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:21,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:56:21,434 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:56:21,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:56:23,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:24,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:56:24,710 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:56:24,710 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:56:24,712 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:56:25,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:25,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:56:26,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:56:26,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:56:27,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:29,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:31,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:31,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:56:31,809 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:56:33,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:35,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:36,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:56:37,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:37,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:56:39,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:41,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:41,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:56:42,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:56:43,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:45,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:46,766 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:56:47,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:47,821 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:56:49,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:51,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:51,437 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:56:51,767 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:56:53,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:53,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:56:55,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:56,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:56:57,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:56:58,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:56:59,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:01,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:01,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:57:03,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:03,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:57:05,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:06,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:57:07,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:09,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:09,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:57:09,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:57:09,194 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:57:09,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:57:09,196 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:57:10,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:57:11,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:11,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:57:13,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:14,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:57:15,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:16,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:57:17,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:19,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:19,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:57:21,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:21,439 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:57:21,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:57:23,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:25,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:25,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:57:26,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:57:27,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:29,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:30,598 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:57:31,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:31,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:57:33,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:35,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:35,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:57:36,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:57:37,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:39,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:41,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:41,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:57:41,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:57:43,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:45,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:46,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:57:46,821 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:57:47,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:49,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:51,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:51,441 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:57:51,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:57:52,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:57:53,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:53,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:57:53,827 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:57:53,828 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:57:53,830 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:57:54,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:57:55,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:56,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:57:57,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:57:57,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:57:59,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:01,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:01,823 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:58:02,694 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:58:03,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:05,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:06,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:58:07,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:08,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:58:09,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:11,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:11,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:58:13,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:13,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:58:15,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:16,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:58:17,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:18,738 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:58:19,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:21,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:21,443 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:58:21,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:58:23,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:24,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:58:25,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:26,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:58:27,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:29,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:29,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:58:31,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:31,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:58:33,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:34,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:58:35,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:36,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:58:37,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:38,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:58:38,373 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:58:38,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:58:38,375 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:58:39,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:39,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:58:40,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:58:41,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:41,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:58:43,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:45,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:45,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:58:46,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:58:47,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:49,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:50,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:58:51,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:51,447 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:58:51,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:58:53,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:55,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:56,168 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:58:56,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:58:57,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:58:59,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:01,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:01,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:59:01,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:59:03,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:05,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:06,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:59:06,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:59:07,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:09,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:11,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:11,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:59:12,228 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:59:13,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:15,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:16,853 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:59:17,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:17,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:59:19,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:21,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:21,450 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:59:21,854 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:59:22,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:59:22,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 22:59:22,942 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 22:59:22,942 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 22:59:22,944 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 22:59:23,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:23,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 22:59:25,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:26,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:59:27,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:28,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:59:29,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:31,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:31,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:59:33,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:33,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:59:35,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:36,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:59:37,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:38,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:59:39,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:41,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:59:41,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:43,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:44,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:59:45,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:46,889 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:59:47,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:49,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:49,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:59:51,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:51,452 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 22:59:51,889 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 22:59:53,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:55,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 22:59:55,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:56,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 22:59:57,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 22:59:59,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:00,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:00:01,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:01,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:00:03,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:05,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:05,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:00:06,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:00:07,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:07,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:00:07,506 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:00:07,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:00:07,508 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:00:08,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:00:09,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:11,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:00:11,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:00:11,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:13,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:15,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:16,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:00:16,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:00:17,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:19,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:21,454 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:00:21,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:00:21,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:21,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:00:23,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:25,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:26,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:00:27,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:00:27,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:29,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:31,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:31,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:00:32,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:00:33,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:35,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:36,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:00:37,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:37,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:00:39,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:41,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:00:41,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:43,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:00:43,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:45,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:46,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:00:47,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:48,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:00:49,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:51,457 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:00:51,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:51,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:00:52,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:00:52,002 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:00:52,003 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:00:52,005 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:00:52,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:00:53,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:53,758 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:00:55,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:56,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:00:57,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:00:59,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:00:59,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:01,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:01,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:01:03,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:04,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:01:05,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:06,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:01:07,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:09,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:09,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:01:11,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:01:11,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:13,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:15,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:01:15,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:16,941 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:01:17,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:19,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:20,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:01:21,459 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:01:21,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:21,941 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:01:23,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:25,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:25,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:01:26,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:01:27,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:29,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:31,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:01:31,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:31,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:01:33,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:35,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:36,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:01:36,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:01:36,579 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:01:36,579 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:01:36,582 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:01:36,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:01:36,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:01:37,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:39,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:41,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:01:41,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:41,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:01:43,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:45,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:46,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:01:47,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:01:47,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:49,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:51,474 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:01:51,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:51,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:01:52,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:01:53,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:55,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:56,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:01:57,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:01:57,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:01:59,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:01,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:01,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:02:03,202 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:02:03,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:05,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:06,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:02:07,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:08,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:02:09,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:11,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:02:11,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:13,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:13,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:02:15,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:16,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:02:17,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:19,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:02:19,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:21,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:02:21,077 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:02:21,077 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:02:21,078 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:02:21,464 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:02:21,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:21,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:02:21,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:02:23,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:24,632 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:02:25,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:26,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:02:27,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:29,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:30,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:02:31,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:31,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:02:33,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:35,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:02:35,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:36,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:02:37,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:39,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:40,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:02:41,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:02:41,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:43,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:45,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:46,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:02:46,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:02:47,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:49,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:51,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:02:51,466 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:02:51,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:51,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:02:53,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:55,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:56,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:02:56,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:02:57,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:02:59,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:01,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:01,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:03:02,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:03:03,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:05,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:03:05,603 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:03:05,603 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:03:05,605 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:03:05,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:05,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:03:06,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:03:07,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:03:07,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:09,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:11,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:03:11,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:12,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:03:13,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:15,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:17,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:03:17,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:18,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:03:19,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:21,467 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:03:21,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:22,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:03:23,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:03:23,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:25,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:26,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:03:27,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:28,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:03:29,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:31,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:32,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:03:33,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:34,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:03:35,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:37,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:03:37,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:39,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:03:39,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:41,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:03:41,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:43,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:44,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:03:45,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:47,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:03:47,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:49,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:50,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:03:50,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:03:50,206 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:03:50,207 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:03:50,208 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:03:50,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:03:51,470 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:03:51,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:52,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:03:53,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:55,512 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:03:55,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:56,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:03:57,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:03:59,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:00,838 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:04:01,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:02,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:04:03,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:05,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:06,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:04:07,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:04:07,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:09,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:11,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:04:11,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:04:11,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:13,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:15,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:16,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:04:17,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:04:17,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:19,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:21,472 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:04:21,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:22,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:04:22,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:04:23,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:25,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:26,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:04:27,466 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:04:27,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:29,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:31,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:32,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:04:32,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:04:33,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:34,649 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:04:34,651 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:04:34,651 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:04:34,653 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:04:34,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:04:35,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:37,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:04:37,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:38,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:04:39,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:41,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:04:41,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:43,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:04:43,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:45,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:47,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:04:47,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:48,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:04:49,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:51,474 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:04:51,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:52,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:04:53,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:54,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:04:55,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:56,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:04:57,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:04:59,597 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:04:59,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:01,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:02,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:05:03,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:04,948 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:05:05,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:07,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:05:07,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:09,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:10,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:05:11,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:05:11,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:13,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:15,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:05:15,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:17,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:05:17,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:19,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:05:19,296 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:05:19,297 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:05:19,299 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:05:19,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:19,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:05:21,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:05:21,477 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:05:21,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:22,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:05:23,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:25,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:26,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:05:26,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:05:27,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:29,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:31,730 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:05:31,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:32,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:05:33,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:35,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:37,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:05:37,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:05:37,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:39,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:41,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:05:41,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:42,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:05:43,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:45,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:47,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:05:47,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:05:47,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:49,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:51,479 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:05:51,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:52,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:05:53,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:05:53,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:55,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:56,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:05:57,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:05:58,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:05:59,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:01,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:02,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:06:03,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:06:03,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:06:03,861 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:06:03,862 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:06:03,863 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:06:03,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:03,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:06:05,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:07,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:06:07,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:09,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:06:09,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:11,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:06:11,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:13,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:14,523 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:06:15,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:17,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:06:17,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:19,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:06:19,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:21,481 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:06:21,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:22,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:06:23,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:25,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:06:25,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:26,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:06:27,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:29,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:30,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:06:31,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:32,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:06:33,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:35,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:06:35,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:37,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:06:37,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:39,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:41,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:06:41,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:06:41,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:43,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:45,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:46,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:06:47,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:06:47,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:48,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:06:48,351 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:06:48,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:06:48,353 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:06:48,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:06:49,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:51,484 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:06:51,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:06:51,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:52,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:06:53,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:55,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:56,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:06:57,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:06:57,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:06:59,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:01,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:02,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:07:02,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:07:02,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:04,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:06,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:07,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:07:07,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:07:08,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:10,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:11,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:07:12,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:13,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:07:14,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:16,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:17,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:07:18,732 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:07:18,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:20,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:21,486 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:07:22,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:07:22,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:24,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:07:24,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:26,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:07:26,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:28,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:29,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:07:30,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:32,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:07:32,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:33,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:07:33,041 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:07:33,041 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:07:33,043 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:07:33,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:07:34,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:07:34,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:36,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:37,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:07:38,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:40,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:07:40,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:41,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:07:42,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:44,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:45,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:07:46,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:47,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:07:48,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:50,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:07:50,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:51,498 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:07:52,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:07:52,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:54,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:56,138 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:07:56,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:07:56,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:07:58,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:01,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:01,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:08:02,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:08:03,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:05,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:06,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:08:07,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:07,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:08:09,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:11,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:11,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:08:12,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:08:13,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:15,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:17,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:17,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:08:17,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:08:17,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:08:17,547 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:08:17,547 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:08:17,549 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:08:18,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:08:19,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:21,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:21,490 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:08:22,107 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:08:22,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:08:23,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:25,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:26,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:08:27,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:28,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:08:29,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:31,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:32,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:08:33,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:33,682 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:08:35,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:37,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:37,143 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:08:39,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:39,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:08:41,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:41,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:08:43,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:44,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:08:45,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:47,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:47,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:08:49,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:49,748 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:08:51,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:51,492 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:08:52,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:08:53,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:55,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:55,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:08:56,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:08:57,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:08:59,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:00,429 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:09:01,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:02,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:09:02,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:09:02,239 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:09:02,239 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:09:02,241 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:09:03,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:03,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:09:05,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:05,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:09:07,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:07,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:09:09,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:11,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:11,125 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:09:11,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:09:13,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:15,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:16,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:09:17,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:17,186 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:09:19,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:21,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:21,494 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:09:21,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:09:22,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:09:23,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:25,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:26,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:09:27,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:27,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:09:29,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:31,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:32,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:09:32,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:09:33,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:35,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:37,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:09:37,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:37,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:09:39,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:41,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:41,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:09:43,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:43,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:09:45,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:46,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:09:46,875 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:09:46,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:09:46,877 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:09:47,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:09:47,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:47,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:09:48,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:09:49,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:51,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:51,507 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:09:52,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:09:53,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:53,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:09:55,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:56,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:09:57,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:59,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:09:59,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:10:01,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:02,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:10:03,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:04,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:10:05,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:07,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:10:07,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:09,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:09,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:10:11,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:11,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:10:13,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:15,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:15,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:10:17,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:10:17,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:19,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:20,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:10:21,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:21,499 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:10:22,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:10:23,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:25,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:26,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:10:26,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:10:27,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:29,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:31,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:31,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:10:31,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:10:31,446 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:10:31,446 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:10:31,447 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:10:32,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:10:32,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:10:33,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:35,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:36,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:10:37,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:10:37,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:39,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:41,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:10:41,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:42,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:10:43,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:45,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:47,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:10:47,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:47,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:10:49,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:51,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:51,501 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:10:52,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:10:52,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:10:53,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:55,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:56,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:10:57,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:10:58,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:10:59,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:01,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:02,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:11:03,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:03,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:11:05,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:07,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:11:07,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:08,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:11:09,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:11,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:11:11,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:13,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:14,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:11:15,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:16,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:11:16,089 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:11:16,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:11:16,092 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:11:16,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:11:17,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:11:17,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:19,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:19,618 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:11:21,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:21,504 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:11:22,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:11:23,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:24,950 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:11:25,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:26,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:11:27,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:29,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:30,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:11:31,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:32,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:11:33,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:35,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:35,620 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:11:37,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:11:37,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:39,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:40,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:11:41,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:11:41,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:43,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:45,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:46,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:11:47,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:11:47,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:49,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:51,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:51,518 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:11:51,519 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:11:52,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:11:53,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:55,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:56,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:11:57,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:11:57,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:11:59,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:00,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:12:00,610 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:12:00,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:12:00,613 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:12:01,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:01,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:12:02,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:12:02,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:12:03,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:05,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:07,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:12:07,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:07,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:12:09,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:11,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:12:11,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:13,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:12:13,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:15,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:17,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:12:17,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:18,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:12:19,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:21,508 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:12:21,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:22,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:12:23,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:23,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:12:25,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:26,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:12:27,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:29,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:12:29,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:31,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:32,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:12:33,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:34,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:12:35,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:37,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:12:37,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:39,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:39,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:12:41,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:12:41,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:43,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:45,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:12:45,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:12:45,083 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:12:45,083 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:12:45,084 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:12:45,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:45,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:12:47,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:12:47,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:49,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:50,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:12:51,510 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:12:51,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:52,338 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:12:53,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:55,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:55,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:12:56,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:12:57,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:12:59,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:01,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:13:01,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:02,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:13:03,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:05,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:06,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:13:07,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:13:07,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:09,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:11,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:13:11,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:11,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:13:13,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:15,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:17,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:13:17,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:13:17,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:19,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:21,513 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:13:21,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:22,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:13:22,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:13:23,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:25,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:26,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:13:27,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:27,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:13:29,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:29,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:13:29,661 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:13:29,661 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:13:29,663 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:13:30,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:13:31,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:32,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:13:33,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:13:33,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:35,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:37,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:13:37,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:38,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:13:39,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:41,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:13:41,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:43,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:43,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:13:45,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:47,355 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:13:47,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:49,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:13:49,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:51,525 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:13:51,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:52,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:13:53,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:54,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:13:55,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:56,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:13:57,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:59,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:13:59,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:14:01,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:02,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:14:03,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:05,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:14:05,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:07,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:14:07,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:09,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:10,663 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:14:11,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:14:11,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:13,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:14,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:14:14,259 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:14:14,260 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:14:14,261 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:14:14,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:14:15,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:16,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:14:16,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:14:17,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:19,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:21,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:14:21,422 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:14:21,517 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:14:21,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:23,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:25,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:26,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:14:26,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:14:27,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:29,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:31,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:14:31,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:32,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:14:33,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:35,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:36,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:14:37,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:14:37,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:39,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:41,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:14:41,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:42,699 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:14:43,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:45,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:46,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:14:47,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:48,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:14:49,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:51,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:14:51,519 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:14:51,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:53,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:14:53,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:55,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:56,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:14:57,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:58,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:14:58,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:14:58,737 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:14:58,737 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:14:58,739 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:14:59,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:14:59,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:15:01,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:15:01,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:03,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:04,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:15:05,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:06,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:15:07,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:09,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:15:09,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:11,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:15:11,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:13,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:14,746 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:15:15,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:16,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:15:17,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:19,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:20,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:15:21,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:15:21,521 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:15:21,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:23,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:25,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:15:25,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:26,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:15:27,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:29,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:30,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:15:31,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:15:31,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:33,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:35,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:36,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:15:36,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:15:37,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:39,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:41,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:15:41,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:15:41,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:43,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:15:43,287 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:15:43,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:15:43,289 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:15:43,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:43,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:15:45,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:46,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:15:46,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:15:47,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:49,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:51,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:15:51,534 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:15:51,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:52,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:15:53,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:55,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:56,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:15:57,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:15:57,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:15:59,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:01,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:16:01,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:02,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:16:03,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:05,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:06,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:16:07,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:08,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:16:09,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:11,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:16:11,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:13,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:16:13,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:15,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:16,517 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:16:17,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:18,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:16:19,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:21,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:16:21,526 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:16:21,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:23,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:24,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:16:25,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:26,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:16:27,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:27,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:16:27,851 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:16:27,851 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:16:27,853 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:16:28,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:16:29,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:16:29,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:31,530 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:16:31,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:33,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:34,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:16:35,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:36,530 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:16:37,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:39,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:40,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:16:41,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:16:41,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:43,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:45,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:16:45,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:46,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:16:47,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:49,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:50,906 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:16:51,528 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:16:51,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:16:51,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:53,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:55,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:56,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:16:56,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:16:57,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:16:59,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:01,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:17:01,605 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:17:01,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:03,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:05,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:06,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:17:06,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:17:07,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:09,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:11,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:17:11,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:12,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:17:12,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:17:12,347 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:17:12,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:17:12,350 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:17:12,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:17:13,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:15,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:16,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:17:17,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:17:17,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:19,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:21,530 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:17:21,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:17:21,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:22,985 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:17:23,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:25,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:26,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:17:27,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:28,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:17:29,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:31,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:17:31,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:33,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:17:33,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:35,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:36,582 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:17:37,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:39,056 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:17:39,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:41,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:17:41,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:43,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:44,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:17:45,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:46,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:17:47,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:49,739 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:17:49,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:51,532 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:17:51,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:17:51,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:53,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:55,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:17:55,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:56,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:17:56,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:17:56,903 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:17:56,903 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:17:56,905 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:17:57,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:17:57,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:17:59,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:00,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:18:01,595 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:18:01,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:03,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:05,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:18:05,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:06,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:18:07,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:09,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:11,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:18:11,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:18:11,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:13,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:15,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:16,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:18:16,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:18:17,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:19,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:21,535 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:18:21,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:18:21,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:18:21,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:23,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:25,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:26,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:18:27,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:18:27,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:29,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:31,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:18:31,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:32,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:18:33,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:35,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:36,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:18:37,836 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:18:37,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:39,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:41,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:18:41,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:18:41,443 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:18:41,443 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:18:41,444 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:18:41,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:41,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:18:43,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:18:43,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:45,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:46,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:18:47,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:48,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:18:49,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:51,537 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:18:51,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:18:51,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:53,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:18:53,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:55,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:56,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:18:57,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:18:59,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:18:59,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:01,683 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:19:01,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:03,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:04,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:19:05,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:06,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:19:07,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:09,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:19:09,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:11,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:19:11,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:13,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:15,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:19:15,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:16,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:19:18,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:20,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:20,594 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:19:21,539 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:19:21,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:19:22,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:24,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:25,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:19:25,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:19:25,996 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:19:25,996 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:19:25,998 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:19:26,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:26,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:19:26,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:19:28,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:30,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:31,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:19:31,715 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:19:32,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:34,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:36,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:36,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:19:36,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:19:38,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:40,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:41,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:19:42,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:19:42,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:44,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:46,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:46,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:19:47,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:19:48,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:50,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:51,551 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:19:51,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:19:52,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:52,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:19:54,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:56,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:19:56,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:19:58,030 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:19:58,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:00,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:01,758 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:20:02,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:03,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:20:04,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:06,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:06,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:20:07,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:08,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:20:09,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:10,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:20:10,534 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:20:10,535 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:20:10,536 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:20:11,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:11,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:20:11,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:20:13,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:14,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:20:15,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:16,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:20:17,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:19,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:19,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:20:21,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:21,544 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:20:21,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:20:23,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:24,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:20:25,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:26,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:20:27,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:29,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:30,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:20:31,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:31,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:20:33,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:35,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:35,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:20:36,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:20:37,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:39,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:40,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:20:41,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:41,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:20:43,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:45,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:46,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:20:46,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:20:47,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:49,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:51,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:51,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:20:51,546 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:20:51,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:20:53,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:54,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:20:54,990 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:20:54,990 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:20:54,992 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:20:55,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:55,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:20:56,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:20:56,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:20:57,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:20:59,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:01,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:01,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:21:02,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:21:03,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:05,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:06,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:21:07,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:07,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:21:09,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:11,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:11,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:21:12,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:21:13,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:15,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:16,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:21:17,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:18,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:21:19,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:21,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:21,548 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:21:21,817 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:21:23,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:23,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:21:25,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:26,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:21:27,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:28,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:21:29,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:31,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:31,819 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:21:33,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:34,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:21:35,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:36,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:21:37,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:39,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:39,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:21:39,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:21:39,588 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:21:39,589 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:21:39,590 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:21:40,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:21:41,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:41,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:21:43,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:44,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:21:45,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:46,838 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:21:47,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:49,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:50,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:21:51,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:51,561 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:21:51,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:21:53,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:55,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:55,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:21:56,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:21:57,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:21:59,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:00,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:22:01,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:01,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:22:03,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:05,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:06,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:22:06,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:22:07,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:09,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:11,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:11,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:22:11,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:22:13,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:15,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:16,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:22:16,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:22:17,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:19,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:21,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:21,552 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:22:21,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:22:22,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:22:23,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:24,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:22:24,104 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:22:24,105 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:22:24,107 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:22:24,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:22:25,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:26,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:22:27,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:27,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:22:29,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:31,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:31,851 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:22:32,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:22:33,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:35,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:36,852 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:22:37,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:38,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:22:39,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:41,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:41,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:22:43,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:43,603 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:22:45,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:46,853 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:22:47,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:48,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:22:49,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:51,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:51,555 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:22:51,854 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:22:53,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:54,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:22:55,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:56,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:22:57,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:59,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:22:59,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:23:01,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:01,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:23:03,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:04,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:23:05,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:06,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:23:07,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:08,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:23:08,565 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:23:08,565 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:23:08,567 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:23:09,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:09,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:23:10,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:23:11,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:23:11,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:13,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:15,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:15,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:23:16,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:23:17,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:19,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:21,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:23:21,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:21,557 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:23:21,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:23:23,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:25,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:26,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:23:26,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:23:27,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:29,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:31,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:31,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:23:31,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:23:33,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:35,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:36,889 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:23:37,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:23:37,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:39,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:41,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:23:41,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:42,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:23:43,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:45,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:46,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:23:47,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:47,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:23:49,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:51,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:51,570 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:23:51,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:23:53,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:23:53,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:23:53,152 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:23:53,152 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:23:53,155 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:23:53,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:53,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:23:55,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:56,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:23:57,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:23:58,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:23:59,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:01,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:01,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:24:03,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:03,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:24:05,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:06,906 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:24:07,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:09,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:24:09,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:11,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:24:11,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:13,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:14,476 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:24:15,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:16,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:24:17,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:19,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:19,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:24:21,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:21,561 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:24:21,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:24:23,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:25,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:24:25,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:26,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:24:27,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:29,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:30,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:24:31,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:31,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:24:33,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:35,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:35,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:24:36,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:24:37,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:37,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:24:37,700 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:24:37,701 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:24:37,702 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:24:38,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:24:39,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:41,235 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:24:41,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:24:41,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:43,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:45,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:46,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:24:46,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:24:47,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:49,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:51,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:51,564 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:24:51,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:24:51,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:24:53,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:55,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:56,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:24:57,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:24:57,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:24:59,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:01,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:01,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:25:02,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:25:03,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:05,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:06,938 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:25:07,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:07,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:25:09,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:11,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:25:11,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:13,233 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:25:13,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:15,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:16,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:25:17,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:18,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:25:19,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:21,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:21,566 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:25:21,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:25:22,166 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:25:22,167 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:25:22,168 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:25:22,170 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:25:22,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:25:23,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:23,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:25:25,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:26,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:25:27,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:29,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:25:29,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:31,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:31,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:25:33,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:34,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:25:35,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:36,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:25:37,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:39,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:40,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:25:41,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:25:41,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:43,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:45,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:25:45,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:46,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:25:47,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:49,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:50,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:25:51,579 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:25:51,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:51,991 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:25:53,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:55,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:56,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:25:56,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:25:57,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:25:59,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:01,369 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:26:01,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:01,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:26:03,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:05,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:06,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:26:06,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:26:06,755 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:26:06,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:26:06,757 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:26:06,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:26:07,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:07,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:26:09,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:11,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:26:11,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:12,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:26:13,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:15,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:17,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:26:17,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:26:17,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:19,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:21,570 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:26:21,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:22,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:26:22,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:26:23,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:25,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:26,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:26:27,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:28,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:26:29,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:31,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:32,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:26:33,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:26:33,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:35,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:37,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:26:37,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:38,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:26:39,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:41,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:26:41,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:43,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:44,085 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:26:45,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:47,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:26:47,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:49,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:26:49,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:51,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:26:51,213 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:26:51,213 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:26:51,215 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:26:51,573 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:26:51,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:51,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:26:52,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:26:53,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:54,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:26:55,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:56,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:26:57,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:26:59,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:00,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:27:01,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:02,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:27:03,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:05,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:27:05,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:07,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:27:07,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:09,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:10,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:27:11,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:27:11,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:13,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:15,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:16,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:27:17,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:27:17,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:19,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:21,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:27:21,575 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:27:21,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:22,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:27:23,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:25,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:26,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:27:26,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:27:27,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:29,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:31,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:32,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:27:32,238 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:27:33,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:35,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:35,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:27:35,836 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:27:35,836 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:27:35,838 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:27:36,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:27:37,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:27:37,590 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:27:37,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:39,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:41,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:27:41,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:42,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:27:43,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:45,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:47,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:27:47,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:48,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:27:49,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:51,588 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:27:51,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:52,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:27:53,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:27:53,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:55,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:56,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:27:57,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:27:58,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:27:59,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:01,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:02,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:28:03,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:04,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:28:05,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:07,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:28:07,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:09,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:28:09,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:11,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:28:11,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:13,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:15,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:28:15,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:17,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:28:17,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:19,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:20,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:28:20,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:28:20,365 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:28:20,366 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:28:20,368 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:28:20,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:28:21,579 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:28:21,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:22,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:28:23,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:25,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:28:25,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:26,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:28:27,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:29,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:30,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:28:31,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:32,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:28:33,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:35,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:36,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:28:37,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:28:37,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:39,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:41,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:28:41,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:28:41,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:43,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:45,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:46,975 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:28:47,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:28:47,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:49,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:51,582 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:28:51,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:52,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:28:52,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:28:53,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:55,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:56,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:28:57,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:28:57,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:28:59,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:01,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:02,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:29:03,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:29:03,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:04,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:29:04,846 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:29:04,847 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:29:04,848 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:29:05,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:05,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:29:07,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:29:07,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:08,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:29:09,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:11,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:29:11,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:13,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:29:13,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:15,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:17,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:29:17,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:19,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:29:19,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:21,585 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:29:21,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:22,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:29:23,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:24,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:29:25,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:26,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:29:27,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:29,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:29:29,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:31,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:32,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:29:33,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:35,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:29:35,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:37,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:29:37,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:39,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:40,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:29:41,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:29:41,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:43,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:45,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:45,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:29:47,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:29:47,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:49,530 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:29:49,531 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:29:49,532 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:29:49,534 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:29:49,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:49,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:29:51,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:29:51,599 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:29:51,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:52,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:29:53,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:55,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:56,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:29:56,627 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:29:57,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:29:59,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:01,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:01,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:30:02,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:30:03,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:05,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:07,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:30:07,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:30:07,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:09,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:11,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:30:11,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:12,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:30:13,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:15,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:17,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:30:17,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:17,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:30:19,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:21,589 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:30:21,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:22,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:30:23,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:30:23,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:26,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:26,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:30:28,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:28,686 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:30:30,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:32,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:32,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:30:34,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:34,045 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:30:34,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:30:34,079 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:30:34,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:30:34,082 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:30:35,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:30:36,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:37,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:30:38,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:39,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:30:40,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:41,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:30:42,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:44,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:44,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:30:46,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:47,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:30:48,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:50,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:50,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:30:51,591 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:30:52,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:52,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:30:54,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:55,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:30:56,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:30:56,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:30:58,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:00,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:00,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:31:02,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:02,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:31:04,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:06,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:06,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:31:07,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:31:08,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:10,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:11,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:31:11,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:31:12,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:14,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:16,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:16,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:31:17,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:31:18,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:18,637 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:31:18,639 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:31:18,640 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:31:18,641 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:31:19,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:31:20,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:21,594 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:31:22,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:22,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:31:22,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:31:24,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:26,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:26,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:31:27,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:31:28,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:30,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:32,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:32,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:31:32,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:31:34,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:36,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:37,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:31:38,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:38,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:31:40,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:41,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:31:42,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:43,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:31:44,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:46,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:47,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:31:48,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:48,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:31:50,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:51,606 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:31:52,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:52,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:31:54,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:54,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:31:56,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:56,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:31:58,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:31:59,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:32:00,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:02,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:02,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:32:03,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:32:03,110 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:32:03,110 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:32:03,112 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:32:03,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:32:04,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:04,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:32:06,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:07,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:32:08,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:10,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:10,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:32:11,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:32:12,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:14,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:15,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:32:16,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:17,202 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:32:18,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:20,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:20,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:32:21,598 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:32:22,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:22,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:32:24,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:26,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:26,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:32:26,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:32:28,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:30,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:31,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:32:32,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:32,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:32:34,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:36,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:36,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:32:37,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:32:38,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:40,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:41,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:32:42,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:42,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:32:44,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:46,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:47,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:32:47,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:32:47,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:32:47,692 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:32:47,692 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:32:47,695 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:32:48,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:48,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:32:50,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:51,600 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:32:52,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:52,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:32:53,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:32:54,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:56,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:56,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:32:58,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:32:58,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:33:00,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:02,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:02,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:33:03,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:33:04,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:06,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:07,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:33:08,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:08,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:33:10,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:11,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:33:12,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:14,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:14,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:33:16,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:17,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:33:18,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:19,692 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:33:20,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:21,602 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:33:22,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:22,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:33:24,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:25,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:33:26,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:26,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:33:28,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:30,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:30,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:33:32,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:33:32,169 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:33:32,169 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:33:32,170 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:33:32,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:32,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:33:32,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:33:33,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:35,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:33:36,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:37,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:33:38,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:40,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:41,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:33:41,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:33:42,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:44,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:46,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:46,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:33:47,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:33:48,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:50,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:51,615 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:33:51,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:33:52,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:33:52,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:54,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:56,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:33:56,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:33:57,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:33:58,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:00,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:02,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:34:02,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:02,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:34:04,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:06,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:07,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:34:07,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:34:08,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:10,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:11,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:34:12,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:13,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:34:14,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:16,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:16,734 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:34:16,735 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:34:16,736 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:34:16,737 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:34:17,235 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:34:17,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:17,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:34:18,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:34:20,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:21,607 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:34:22,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:34:22,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:23,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:34:24,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:26,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:26,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:34:28,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:29,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:34:30,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:32,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:34:32,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:34,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:34,519 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:34:36,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:37,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:34:38,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:39,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:34:40,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:41,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:34:42,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:44,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:45,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:34:46,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:47,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:34:48,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:50,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:50,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:34:51,609 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:34:52,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:34:52,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:54,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:55,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:34:56,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:34:56,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:34:58,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:00,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:01,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:35:01,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:35:01,300 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:35:01,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:35:01,302 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:35:01,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:35:02,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:35:02,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:03,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:06,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:06,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:35:07,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:35:08,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:10,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:11,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:35:11,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:35:12,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:14,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:16,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:17,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:35:17,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:35:18,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:20,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:21,612 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:35:22,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:35:22,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:22,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:35:24,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:26,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:26,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:35:27,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:35:28,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:30,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:32,263 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:35:32,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:33,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:35:34,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:36,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:37,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:35:38,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:38,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:35:40,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:41,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:35:42,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:43,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:35:44,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:45,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:35:45,719 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:35:45,720 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:35:45,722 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:35:46,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:46,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:35:47,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:35:47,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:49,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:35:50,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:51,627 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:35:52,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:35:52,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:54,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:54,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:35:56,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:56,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:35:58,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:35:59,941 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:36:00,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:02,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:36:02,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:04,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:05,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:36:06,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:07,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:36:08,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:10,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:10,690 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:36:11,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:36:12,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:14,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:16,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:36:16,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:17,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:36:18,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:20,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:21,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:36:21,616 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:36:22,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:36:22,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:24,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:26,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:26,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:36:26,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:36:28,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:30,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:36:30,336 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:36:30,336 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:36:30,338 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:36:30,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:30,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:36:31,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:32,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:36:32,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:36:34,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:36,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:37,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:36:37,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:36:38,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:40,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:41,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:36:42,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:42,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:36:43,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:45,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:47,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:36:47,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:48,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:36:49,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:51,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:51,619 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:36:52,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:36:53,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:53,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:36:55,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:56,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:36:57,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:36:58,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:36:59,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:01,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:02,275 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:37:03,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:04,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:37:05,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:07,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:37:07,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:09,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:09,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:37:11,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:37:11,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:13,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:14,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:37:14,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:37:14,910 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:37:14,910 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:37:14,912 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:37:15,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:15,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:37:17,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:37:17,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:19,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:20,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:37:21,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:21,621 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:37:22,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:37:23,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:25,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:25,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:37:26,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:37:27,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:29,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:30,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:37:31,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:32,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:37:33,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:35,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:36,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:37:37,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:37:37,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:39,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:41,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:37:41,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:41,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:37:43,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:45,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:46,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:37:47,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:37:47,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:49,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:51,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:51,632 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:37:52,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:37:52,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:37:53,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:55,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:56,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:37:57,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:37:57,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:59,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:37:59,316 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:37:59,316 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:37:59,318 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:37:59,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:37:59,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:38:01,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:02,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:38:02,852 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:38:03,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:05,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:07,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:38:07,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:08,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:38:09,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:11,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:38:11,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:13,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:13,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:38:15,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:17,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:38:17,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:18,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:38:19,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:21,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:21,626 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:38:22,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:38:23,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:24,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:38:25,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:26,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:38:27,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:29,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:29,629 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:38:31,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:32,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:38:33,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:34,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:38:35,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:37,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:38:37,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:39,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:40,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:38:41,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:38:41,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:43,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:43,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:38:43,864 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:38:43,864 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:38:43,865 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:38:44,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:38:45,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:45,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:38:47,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:38:47,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:49,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:50,961 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:38:51,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:51,628 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:38:52,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:38:53,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:55,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:56,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:38:56,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:38:57,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:38:59,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:01,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:01,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:39:02,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:39:03,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:05,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:06,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:39:07,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:39:07,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:09,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:11,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:39:11,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:12,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:39:13,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:15,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:17,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:39:17,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:17,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:39:19,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:21,631 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:39:21,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:22,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:39:22,632 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:39:23,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:25,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:26,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:39:27,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:28,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:39:28,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:39:28,341 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:39:28,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:39:28,343 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:39:28,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:39:29,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:31,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:32,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:39:33,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:33,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:39:35,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:37,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:39:37,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:39,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:39:39,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:41,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:39:41,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:43,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:44,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:39:45,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:47,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:39:47,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:49,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:49,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:39:51,642 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:39:51,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:52,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:39:53,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:55,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:39:55,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:56,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:39:57,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:39:59,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:00,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:40:01,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:02,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:40:03,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:05,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:05,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:40:07,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:40:07,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:09,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:11,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:40:11,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:40:11,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:13,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:40:13,071 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:40:13,071 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:40:13,073 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:40:13,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:13,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:40:15,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:16,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:40:17,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:40:17,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:19,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:21,635 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:40:21,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:40:21,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:22,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:40:23,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:25,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:26,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:40:27,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:40:27,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:29,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:31,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:32,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:40:32,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:40:33,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:35,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:37,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:40:37,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:37,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:40:39,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:41,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:40:41,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:43,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:40:43,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:45,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:47,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:40:47,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:48,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:40:49,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:51,637 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:40:51,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:52,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:40:53,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:53,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:40:55,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:56,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:40:57,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:40:57,567 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:40:57,567 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:40:57,569 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:40:57,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:40:57,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:40:59,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:40:59,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:01,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:02,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:41:03,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:04,688 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:41:05,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:07,338 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:41:07,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:09,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:10,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:41:11,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:41:11,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:13,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:15,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:41:15,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:17,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:41:17,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:19,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:20,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:41:21,639 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:41:21,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:22,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:41:23,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:25,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:26,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:41:26,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:41:27,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:29,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:31,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:41:31,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:32,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:41:33,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:35,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:36,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:41:37,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:41:37,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:39,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:41,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:41:41,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:42,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:41:42,166 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:41:42,167 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:41:42,167 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:41:42,169 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:41:42,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:41:43,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:45,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:47,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:41:47,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:41:47,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:49,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:51,658 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:41:51,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:52,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:41:52,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:41:53,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:55,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:56,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:41:57,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:41:58,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:41:59,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:01,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:02,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:42:03,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:42:03,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:05,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:07,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:42:07,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:08,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:42:09,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:11,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:42:11,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:13,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:14,097 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:42:15,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:17,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:42:17,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:19,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:42:19,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:21,643 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:42:21,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:22,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:42:23,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:24,758 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:42:25,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:26,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:42:26,586 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:42:26,587 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:42:26,587 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:42:26,590 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:42:26,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:42:27,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:29,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:30,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:42:31,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:32,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:42:33,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:35,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:42:35,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:37,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:42:37,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:39,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:40,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:42:41,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:42:41,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:43,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:45,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:46,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:42:47,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:42:47,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:49,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:51,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:42:51,646 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:42:51,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:52,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:42:53,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:55,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:56,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:42:56,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:42:57,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:42:59,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:01,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:02,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:43:02,429 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:43:03,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:05,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:07,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:43:07,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:43:07,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:09,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:11,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:43:11,027 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:43:11,028 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:43:11,030 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:43:11,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:43:11,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:11,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:43:12,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:43:13,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:15,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:17,431 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:43:18,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:18,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:43:20,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:21,648 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:43:22,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:22,432 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:43:23,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:43:24,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:26,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:26,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:43:28,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:28,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:43:30,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:32,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:32,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:43:34,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:34,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:43:36,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:37,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:43:38,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:39,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:43:40,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:41,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:43:42,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:44,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:44,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:43:46,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:46,449 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:43:48,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:50,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:50,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:43:51,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:43:51,660 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:43:52,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:54,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:55,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:43:55,555 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:43:55,556 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:43:55,557 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:43:55,558 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:43:56,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:43:56,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:43:56,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:43:58,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:00,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:00,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:44:01,476 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:44:02,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:04,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:06,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:06,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:44:06,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:44:08,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:10,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:11,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:44:11,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:44:12,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:14,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:16,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:16,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:44:16,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:44:18,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:20,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:21,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:44:21,652 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:44:22,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:22,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:44:24,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:26,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:26,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:44:27,688 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:44:28,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:30,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:31,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:44:32,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:33,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:44:34,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:36,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:36,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:44:38,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:38,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:44:40,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:40,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:44:40,160 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:44:40,160 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:44:40,162 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:44:41,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:44:41,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:44:42,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:43,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:44:44,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:46,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:46,536 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:44:48,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:49,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:44:50,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:51,537 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:44:51,655 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:44:52,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:54,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:54,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:44:56,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:56,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:44:58,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:44:59,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:45:00,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:01,538 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:45:02,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:04,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:05,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:45:06,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:06,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:45:08,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:10,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:10,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:45:11,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:45:12,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:14,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:15,836 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:45:16,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:16,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:45:18,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:20,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:21,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:45:21,555 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:45:21,657 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:45:22,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:24,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:24,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:45:24,773 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:45:24,773 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:45:24,774 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:45:25,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:45:26,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:26,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:45:26,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:45:28,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:30,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:31,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:45:31,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:45:32,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:34,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:36,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:36,588 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:45:37,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:45:38,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:40,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:41,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:45:42,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:42,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:45:44,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:46,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:46,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:45:47,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:45:48,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:50,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:51,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:45:51,670 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:45:52,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:53,262 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:45:54,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:56,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:56,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:45:58,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:45:58,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:46:00,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:46:01,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:46:02,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:46:03,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:46:04,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:46:06,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:46:06,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:46:08,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:46:09,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:46:09,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:46:09,332 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:46:09,332 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:46:09,334 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:46:10,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:46:10,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:46:10,896 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:46:10,898 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:46:10,899 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:46:10,902 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:46:11,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:46:11,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:46:12,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:46:14,898 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:46:16,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:46:17,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:46:19,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:46:21,662 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:46:22,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:46:24,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:46:24,976 DEBUG SenderThread:156911 [sender.py:send():379] send: telemetry -2024-04-25 23:46:24,977 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:46:24,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:46:24,980 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:46:24,981 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:46:24,982 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:46:24,982 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:46:24,983 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:46:24,983 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:46:24,984 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:46:24,984 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:46:24,985 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:46:24,986 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:46:24,986 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:46:24,987 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:46:25,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:46:26,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:46:26,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:46:28,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:46:29,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:46:32,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:46:34,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:46:37,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:46:39,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:46:41,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:46:44,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:46:47,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/config.yaml -2024-04-25 23:46:47,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:46:51,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:46:51,664 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:46:52,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:46:56,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:46:56,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:47:01,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:47:02,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:47:06,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:47:07,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:47:11,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:47:11,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:47:16,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:47:17,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:47:21,588 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history -2024-04-25 23:47:21,590 DEBUG SenderThread:156911 [sender.py:send():379] send: history -2024-04-25 23:47:21,591 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record -2024-04-25 23:47:21,593 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-25 23:47:21,676 DEBUG SenderThread:156911 [sender.py:send():379] send: stats -2024-04-25 23:47:21,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:47:22,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:47:22,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json -2024-04-25 23:47:22,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:47:24,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:47:26,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:47:27,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:47:28,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:47:32,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:47:32,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:47:33,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:47:36,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:47:37,260 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive -2024-04-25 23:47:38,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log -2024-04-25 23:47:38,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report -2024-04-25 23:47:41,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-25 23:47:42,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:49:16,511 INFO StreamThr :212584 [internal.py:wandb_internal():86] W&B internal server running at pid: 212584, started at: 2024-04-25 23:49:16.510614 +2024-04-25 23:49:16,512 DEBUG HandlerThread:212584 [handler.py:handle_request():146] handle_request: status +2024-04-25 23:49:16,519 INFO WriterThread:212584 [datastore.py:open_for_write():87] open: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/run-ozdw63qu.wandb +2024-04-25 23:49:16,520 DEBUG SenderThread:212584 [sender.py:send():379] send: header +2024-04-25 23:49:16,537 DEBUG SenderThread:212584 [sender.py:send():379] send: run +2024-04-25 23:49:16,772 INFO SenderThread:212584 [dir_watcher.py:__init__():211] watching files in: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/files +2024-04-25 23:49:16,772 INFO SenderThread:212584 [sender.py:_start_run_threads():1124] run started: ozdw63qu with start time 1714088956.516191 +2024-04-25 23:49:16,778 DEBUG HandlerThread:212584 [handler.py:handle_request():146] handle_request: check_version +2024-04-25 23:49:16,778 DEBUG SenderThread:212584 [sender.py:send_request():406] send_request: check_version +2024-04-25 23:49:16,833 DEBUG HandlerThread:212584 [handler.py:handle_request():146] handle_request: run_start +2024-04-25 23:49:16,892 DEBUG HandlerThread:212584 [system_info.py:__init__():26] System info init +2024-04-25 23:49:16,892 DEBUG HandlerThread:212584 [system_info.py:__init__():41] System info init done +2024-04-25 23:49:16,892 INFO HandlerThread:212584 [system_monitor.py:start():194] Starting system monitor +2024-04-25 23:49:16,893 INFO SystemMonitor:212584 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-25 23:49:16,893 INFO HandlerThread:212584 [system_monitor.py:probe():214] Collecting system info +2024-04-25 23:49:16,893 INFO SystemMonitor:212584 [interfaces.py:start():190] Started cpu monitoring +2024-04-25 23:49:16,893 INFO SystemMonitor:212584 [interfaces.py:start():190] Started disk monitoring +2024-04-25 23:49:16,894 INFO SystemMonitor:212584 [interfaces.py:start():190] Started gpu monitoring +2024-04-25 23:49:16,894 INFO SystemMonitor:212584 [interfaces.py:start():190] Started memory monitoring +2024-04-25 23:49:16,895 INFO SystemMonitor:212584 [interfaces.py:start():190] Started network monitoring +2024-04-25 23:49:16,939 DEBUG HandlerThread:212584 [system_info.py:probe():150] Probing system +2024-04-25 23:49:16,942 DEBUG HandlerThread:212584 [system_info.py:_probe_git():135] Probing git +2024-04-25 23:49:16,961 DEBUG HandlerThread:212584 [system_info.py:_probe_git():143] Probing git done +2024-04-25 23:49:16,961 DEBUG HandlerThread:212584 [system_info.py:probe():198] Probing system done +2024-04-25 23:49:16,961 DEBUG HandlerThread:212584 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1048-aws-x86_64-with-glibc2.31', 'python': '3.11.9', 'heartbeatAt': '2024-04-25T23:49:16.939535', 'startedAt': '2024-04-25T23:49:16.496864', 'docker': None, 'cuda': None, 'args': ('./config_full.yaml',), 'state': 'running', 'program': '/fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/./run_sft.py', 'codePathLocal': 'run_sft.py', 'codePath': 'run_sft.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/distil-zephyr-1.5b-ssft-ultrachat', 'commit': 'cbea69c6b95c970317a1e47c3f614b55b33f8ed9'}, 'email': None, 'root': '/fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat', 'host': 'ip-26-0-167-177', 'username': 'sanchit', 'executable': '/fsx/sanchit/miniconda3/envs/alignment/bin/python', 'cpu_count': 96, 'cpu_count_logical': 96, 'cpu_freq': {'current': 2718.7540416666648, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2603.523, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3596.748, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3596.723, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3597.872, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3590.698, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2688.441, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3582.227, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3596.99, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3597.34, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 290.7472343444824, 'used': 58.58917236328125}}, 'gpu': 'NVIDIA H100 80GB HBM3', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}], 'memory': {'total': 1999.9855155944824}} +2024-04-25 23:49:16,962 INFO HandlerThread:212584 [system_monitor.py:probe():224] Finished collecting system info +2024-04-25 23:49:16,962 INFO HandlerThread:212584 [system_monitor.py:probe():227] Publishing system info +2024-04-25 23:49:16,962 DEBUG HandlerThread:212584 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-04-25 23:49:17,774 INFO Thread-12 :212584 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/files/conda-environment.yaml +2024-04-25 23:49:20,884 DEBUG HandlerThread:212584 [system_info.py:_save_conda():222] Saving conda packages done +2024-04-25 23:49:20,887 INFO HandlerThread:212584 [system_monitor.py:probe():229] Finished publishing system info +2024-04-25 23:49:20,915 DEBUG SenderThread:212584 [sender.py:send():379] send: files +2024-04-25 23:49:20,915 INFO SenderThread:212584 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now +2024-04-25 23:49:21,061 DEBUG HandlerThread:212584 [handler.py:handle_request():146] handle_request: python_packages +2024-04-25 23:49:21,061 DEBUG SenderThread:212584 [sender.py:send_request():406] send_request: python_packages +2024-04-25 23:49:21,061 DEBUG HandlerThread:212584 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:49:21,062 DEBUG HandlerThread:212584 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 23:49:21,063 DEBUG SenderThread:212584 [sender.py:send_request():406] send_request: stop_status +2024-04-25 23:49:21,184 INFO wandb-upload_0:212584 [upload_job.py:push():131] Uploaded file /tmp/tmp9wuipw18wandb/r4fodxap-wandb-metadata.json +2024-04-25 23:49:21,185 DEBUG SenderThread:212584 [sender.py:send():379] send: telemetry +2024-04-25 23:49:21,185 DEBUG SenderThread:212584 [sender.py:send():379] send: config +2024-04-25 23:49:21,187 DEBUG SenderThread:212584 [sender.py:send():379] send: metric +2024-04-25 23:49:21,188 DEBUG SenderThread:212584 [sender.py:send():379] send: telemetry +2024-04-25 23:49:21,188 DEBUG SenderThread:212584 [sender.py:send():379] send: metric +2024-04-25 23:49:21,188 WARNING SenderThread:212584 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-04-25 23:49:21,188 DEBUG SenderThread:212584 [sender.py:send():379] send: telemetry +2024-04-25 23:49:21,778 INFO Thread-12 :212584 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/files/conda-environment.yaml +2024-04-25 23:49:21,778 INFO Thread-12 :212584 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/files/wandb-metadata.json +2024-04-25 23:49:21,778 INFO Thread-12 :212584 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/files/requirements.txt +2024-04-25 23:49:21,778 INFO Thread-12 :212584 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/files/output.log +2024-04-25 23:49:22,189 DEBUG HandlerThread:212584 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:49:23,780 INFO Thread-12 :212584 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/files/output.log +2024-04-25 23:49:24,232 DEBUG SenderThread:212584 [sender.py:send():379] send: telemetry +2024-04-25 23:49:24,233 DEBUG SenderThread:212584 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:49:24,234 DEBUG HandlerThread:212584 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:49:24,236 INFO SenderThread:212584 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:49:24,237 DEBUG SenderThread:212584 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:49:24,238 INFO SenderThread:212584 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:49:24,238 DEBUG SenderThread:212584 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:49:24,239 INFO SenderThread:212584 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:49:24,239 DEBUG SenderThread:212584 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:49:24,240 INFO SenderThread:212584 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:49:24,240 DEBUG SenderThread:212584 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:49:24,242 INFO SenderThread:212584 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:49:24,242 DEBUG SenderThread:212584 [sender.py:send():379] send: metric +2024-04-25 23:49:24,242 DEBUG SenderThread:212584 [sender.py:send():379] send: history +2024-04-25 23:49:24,242 DEBUG SenderThread:212584 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:49:24,243 INFO SenderThread:212584 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:49:24,782 INFO Thread-12 :212584 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/files/wandb-summary.json +2024-04-25 23:49:25,754 DEBUG HandlerThread:212584 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:49:25,757 DEBUG SenderThread:212584 [sender.py:send():379] send: metric +2024-04-25 23:49:25,758 DEBUG SenderThread:212584 [sender.py:send():379] send: metric +2024-04-25 23:49:25,758 DEBUG SenderThread:212584 [sender.py:send():379] send: metric +2024-04-25 23:49:25,758 DEBUG SenderThread:212584 [sender.py:send():379] send: metric +2024-04-25 23:49:25,758 DEBUG SenderThread:212584 [sender.py:send():379] send: history +2024-04-25 23:49:25,758 DEBUG SenderThread:212584 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:49:25,760 INFO SenderThread:212584 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:49:25,783 INFO Thread-12 :212584 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/files/wandb-summary.json +2024-04-25 23:49:25,784 INFO Thread-12 :212584 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/files/output.log +2024-04-25 23:49:26,785 INFO Thread-12 :212584 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/files/output.log +2024-04-25 23:49:27,764 DEBUG HandlerThread:212584 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:49:27,787 INFO Thread-12 :212584 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/files/output.log +2024-04-25 23:49:31,791 INFO Thread-12 :212584 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/files/output.log +2024-04-25 23:49:33,688 DEBUG HandlerThread:212584 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:49:36,062 DEBUG HandlerThread:212584 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:49:36,063 DEBUG HandlerThread:212584 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 23:49:36,063 DEBUG SenderThread:212584 [sender.py:send_request():406] send_request: stop_status +2024-04-25 23:49:37,798 INFO Thread-12 :212584 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_234916-ozdw63qu/files/output.log